From 22780b87dcaa8f7b1540fcaeebd99815827c0495 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:34:44 -0700 Subject: [PATCH 1/9] Bump appwrite version to 1.4.0 --- README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README-CN.md b/README-CN.md index b634458547..b4ce78b1f3 100644 --- a/README-CN.md +++ b/README-CN.md @@ -66,7 +66,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` ### Windows @@ -78,7 +78,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` #### PowerShell @@ -88,7 +88,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 2b82d49eb0..2fb6fb1351 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` ### Windows @@ -87,7 +87,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` #### PowerShell @@ -97,7 +97,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 646a9d1cce..228fc4809e 100644 --- a/app/init.php +++ b/app/init.php @@ -107,8 +107,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 506; -const APP_VERSION_STABLE = '1.3.8'; +const APP_CACHE_BUSTER = 507; +const APP_VERSION_STABLE = '1.4.0'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; From 586ad1984cf9d7d1470de415fbebf28e31c5aad4 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:36:10 -0700 Subject: [PATCH 2/9] Update 1.4 migration --- src/Appwrite/Migration/Version/V19.php | 419 ++++++++++++++++++++++++- 1 file changed, 409 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 39522aff0b..44d0b308d5 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -2,8 +2,6 @@ namespace Appwrite\Migration\Version; -use Appwrite\Auth\Auth; -use Utopia\Config\Config; use Appwrite\Migration\Migration; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -76,24 +74,349 @@ class V19 extends Migration */ private function migrateCollections(): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + $collectionType = match ($internalProjectId) { + 'console' => 'console', + default => 'projects', + }; + $collections = $this->collections[$collectionType]; + foreach ($collections as $collection) { $id = $collection['$id']; + + if ($id === 'schedules' && $internalProjectId === 'console') { + continue; + } + Console::log("Migrating Collection \"{$id}\""); $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); switch ($id) { - case 'projects': + case '_metadata': + $this->createCollection('identities'); + $this->createCollection('migrations'); + $this->createCollection('statsLogger'); // TODO: should we do this now? + break; + case 'attributes': + case 'indexes': try { - /** - * Create 'passwordHistory' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'smtp'); - $this->createAttributeFromCollection($this->projectDB, $id, 'templates'); + $this->projectDB->updateAttribute($id, 'databaseInternalId', required: true); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'SMTP and Templates' from {$id}: {$th->getMessage()}"); + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'collectionInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'error'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'error' from {$id}: {$th->getMessage()}"); + } + break; + case 'builds': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + $this->projectDB->deleteCachedCollection($id); + + // TODO: stderr and stdout => logs + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'certificates': + try { + $this->projectDB->renameAttribute($id, 'log', 'logs'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'logs', size: 1000000); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + break; + case 'databases': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'enabled'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'enabled' from {$id}: {$th->getMessage()}"); + } + break; + case 'deployments': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'buildInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'buildInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'executions': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stderr', 'errors'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stdout', 'logs'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'logs' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'statusCode', 'responseStatusCode'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'responseStatusCode' from {$id}: {$th->getMessage()}"); + } + break; + case 'functions': + $attributesToCreate = [ + 'live', + 'installationId', + 'installationInternalId', + 'providerRepositoryId', + 'repositoryId', + 'repositoryInternalId', + 'providerBranch', + 'providerRootDirectory', + 'providerSilentMode', + 'logging', + 'deploymentInternalId', + 'scheduleInternalId', + 'scheduleId', + 'version', + 'entrypoint', + 'commands', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + '_key_installationId', + '_key_installationInternalId', + '_key_providerRepositoryId', + '_key_repositoryId', + '_key_repositoryInternalId', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + break; + case 'memberships': + try { + $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + // Intentional fall through to update memberships.userInternalId + case 'sessions': + case 'tokens': + try { + $this->projectDB->updateAttribute($id, 'userInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'domains': + case 'keys': + case 'platforms': + case 'webhooks': + try { + $this->projectDB->updateAttribute($id, 'projectInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'projects': + $attributesToCreate = [ + 'database', + 'enabled', + 'smtp', + 'templates', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + } + } + + // TODO: delete domains? + break; + case 'schedules': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'stats': + // TODO: should we do this now? + try { + $this->projectDB->updateAttribute($id, 'value', signed: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteAttribute($id, 'type'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteIndex($id, '_key_metric_period_time'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'teams': + try { + $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'users': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'labels'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'labels' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'accessedAt' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'search', filters: ['userSearch']); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'search' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_accessedAt'); + } catch (\Throwable $th) { + Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}"); + } + break; + case 'variables': + try { + $this->projectDB->deleteIndex($id, '_key_function'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteIndex($id, '_key_uniqueKey'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceType'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceType' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionInternalId', 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionId', 'resourceId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + $indexesToCreate = [ + '_key_resourceInternalId', + '_key_resourceId', + '_key_resourceType', + '_key_uniqueKey', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } } break; default: @@ -117,6 +440,69 @@ class V19 extends Migration protected function fixDocument(Document $document): Document { switch ($document->getCollection()) { + case '_metadata': + // TODO: function schedules + + // TODO: migrate statsLogger? + break; + case 'attributes': + case 'indexes': + $status = $document->getAttribute('status', ''); + if ($status === 'failed') { + $document->setAttribute('error', 'Unknown problem'); + } + break; + case 'builds': + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + break; + case 'collections': + case 'databases': + $document->setAttribute('enabled', true); + break; + case 'deployments': + $resourceId = $document->getAttribute('resourceId'); + $function = $this->projectDB->getDocument('functions', $resourceId); + $document->setAttribute('resourceInternalId', $function->getInternalId()); + + $buildId = $document->getAttribute('buildId'); + $build = $this->projectDB->getDocument('builds', $buildId); + $document->setAttribute('buildInternalId', $build->getInternalId()); + + $document->setAttribute('type', 'manual'); + break; + case 'executions': + $functionId = $document->getAttribute('functionId'); + $function = $this->projectDB->getDocument('functions', $functionId); + $document->setAttribute('functionInternalId', $function->getInternalId()); + + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + break; + case 'functions': + $document->setAttribute('live', true); + $document->setAttribute('logging', true); + $document->setAttribute('version', 'v2'); + $deploymentId = $document->getAttribute('deployment'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + + $schedule = $this->consoleDB->createDocument('schedules', new Document([ + 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'resourceType' => 'function', + 'resourceId' => $document->getId(), + 'resourceInternalId' => $document->getInternalId(), + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $this->project->getId(), + 'schedule' => $document->getAttribute('schedule'), + 'active' => !empty($document->getAttribute('schedule')) && !empty($document->getAttribute('deployment')), + ])); + + $document->setAttribute('scheduleId', $schedule->getId()); + $document->setAttribute('scheduleInternalId', $schedule->getInternalId()); + break; case 'projects': /** * Bump version number. @@ -126,6 +512,12 @@ class V19 extends Migration $document->setAttribute('smtp', []); $document->setAttribute('templates', []); + break; + case 'rules': + // TODO: convert domains + + break; + default: break; } @@ -173,6 +565,13 @@ class V19 extends Migration Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); $this->alterPermissionIndex($id); $this->alterUidType($id); + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); + } } } } From 436600547c1772f718cf4dc1546759a6c456cb8d Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:37:16 -0700 Subject: [PATCH 3/9] Fix Migration task after it was converted to a Platform task --- src/Appwrite/Migration/Migration.php | 41 ++++++++++++++++++++----- src/Appwrite/Migration/Version/V19.php | 36 ++++++++-------------- src/Appwrite/Platform/Tasks/Migrate.php | 34 +++++++++----------- 3 files changed, 61 insertions(+), 50 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 9ea3443091..9c6fbe6b39 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -77,7 +77,11 @@ abstract class Migration Authorization::disable(); Authorization::setDefaultStatus(false); - $this->collections = array_merge([ + $this->collections = Config::getParam('collections', []); + + $projectCollections = $this->collections['projects']; + + $this->collections['projects'] = array_merge([ '_metadata' => [ '$id' => ID::custom('_metadata'), '$collection' => Database::METADATA @@ -90,7 +94,7 @@ abstract class Migration '$id' => ID::custom('abuse'), '$collection' => Database::METADATA ] - ], Config::getParam('collections', [])); + ], $projectCollections); } /** @@ -131,7 +135,14 @@ abstract class Migration */ public function forEachDocument(callable $callback): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + + $collections = match ($internalProjectId) { + 'console' => $this->collections['console'], + default => $this->collections['projects'], + }; + + foreach ($collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { continue; } @@ -237,10 +248,15 @@ abstract class Migration { $name ??= $id; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) { $attributes = []; $indexes = []; - $collection = $this->collections[$id]; + $collection = $this->collections[$collectionType][$id]; foreach ($collection['attributes'] as $attribute) { $attributes[] = new Document([ @@ -286,9 +302,16 @@ abstract class Migration public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$from] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if ($from === 'files') { + $collectionType = 'buckets'; + } + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { - throw new Exception("Collection {$collectionId} not found"); + throw new Exception("Collection {$from} not found"); } $attributes = $collection['attributes']; @@ -332,7 +355,11 @@ abstract class Migration public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$collectionId] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 44d0b308d5..74637c42ad 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -3,8 +3,11 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; +use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; class V19 extends Migration @@ -90,7 +93,7 @@ class V19 extends Migration Console::log("Migrating Collection \"{$id}\""); - $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + $this->projectDB->setNamespace("_$internalProjectId"); switch ($id) { case '_metadata': @@ -294,28 +297,20 @@ class V19 extends Migration // TODO: delete domains? break; - case 'schedules': - try { - $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); - } - break; case 'stats': // TODO: should we do this now? try { $this->projectDB->updateAttribute($id, 'value', signed: true); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'value' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->deleteAttribute($id, 'type'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'type' from {$id}: {$th->getMessage()}"); } try { @@ -329,15 +324,7 @@ class V19 extends Migration $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); - } - break; - case 'teams': - try { - $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); } break; case 'users': @@ -441,8 +428,6 @@ class V19 extends Migration { switch ($document->getCollection()) { case '_metadata': - // TODO: function schedules - // TODO: migrate statsLogger? break; case 'attributes': @@ -488,6 +473,7 @@ class V19 extends Migration $deploymentId = $document->getAttribute('deployment'); $deployment = $this->projectDB->getDocument('deployments', $deploymentId); $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + $document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint')); $schedule = $this->consoleDB->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region @@ -509,6 +495,10 @@ class V19 extends Migration */ $document->setAttribute('version', '1.4.0'); + $databases = Config::getParam('pools-database', []); + $database = $databases[0]; + + $document->setAttribute('database', $database); $document->setAttribute('smtp', []); $document->setAttribute('templates', []); @@ -567,7 +557,7 @@ class V19 extends Migration $this->alterUidType($id); try { - $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 90b4234109..38e5c8aa46 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -7,10 +7,10 @@ use Utopia\CLI\Console; use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -26,20 +26,22 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') /** @TODO APP_VERSION_STABLE needs to be defined */ ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->inject('register') - ->callback(fn ($version, $register) => $this->action($version, $register)); + ->inject('cache') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); } - private function clearProjectsCache(Redis $redis, Document $project) + private function clearProjectsCache(Cache $cache, Document $project) { try { - $redis->del($redis->keys("cache-_{$project->getInternalId()}:*")); + $cache->purge("cache-_{$project->getInternalId()}:*"); } catch (\Throwable $th) { Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); } } - public function action(string $version, Registry $register) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -52,14 +54,6 @@ class Migrate extends Action Console::success('Starting Data Migration to version ' . $version); - $dbPool = $register->get('dbPool', true); - $redis = $register->get('cache', true); - - $cache = new Cache(new RedisCache($redis)); - - $dbForConsole = $dbPool->getDB('console', $cache); - $dbForConsole->setNamespace('_project_console'); - $console = $app->getResource('console'); $limit = 30; @@ -79,6 +73,7 @@ class Migrate extends Action } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + /** @var Migration $migration */ $migration = new $class(); while (!empty($projects)) { @@ -90,11 +85,11 @@ class Migrate extends Action continue; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); try { // TODO: Iterate through all project DBs - $projectDB = $dbPool->getDB($project->getId(), $cache); + $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) ->execute(); @@ -103,11 +98,11 @@ class Migrate extends Action throw $th; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); } $sum = \count($projects); - $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); + $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]); $offset = $offset + $limit; $count = $count + $sum; @@ -115,7 +110,6 @@ class Migrate extends Action Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); } - Swoole\Event::wait(); // Wait for Coroutines to finish Console::success('Data Migration Completed'); } } From 9f6b27f828d726131a971956622668c2626a30c8 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:32:32 -0700 Subject: [PATCH 4/9] Uncomment new db env vars --- app/config/variables.php | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 4187dc69bd..23af28f56d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -342,24 +342,24 @@ return [ 'question' => '', 'filter' => '' ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_PROJECT', -// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_CONSOLE', -// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ] + [ + 'name' => '_APP_CONNECTIONS_DB_PROJECT', + 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ] ], ], [ From e8f522975689fbcb5bd0bb924585d8c7ce2e0f16 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:34:54 -0700 Subject: [PATCH 5/9] Fix console db namespace --- app/cli.php | 2 +- app/init.php | 2 +- app/realtime.php | 2 +- app/worker.php | 2 +- src/Appwrite/Resque/Worker.php | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index d1dd885775..b410ab2ac7 100644 --- a/app/cli.php +++ b/app/cli.php @@ -58,7 +58,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { ->getResource(); $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); + $dbForConsole->setNamespace('_console'); // Ensure tables exist $collections = Config::getParam('collections', [])['console']; diff --git a/app/init.php b/app/init.php index 228fc4809e..3aadab7c0d 100644 --- a/app/init.php +++ b/app/init.php @@ -1090,7 +1090,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $database = new Database($dbAdapter, $cache); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; }, ['pools', 'cache']); diff --git a/app/realtime.php b/app/realtime.php index 772eee49d6..25b0532b42 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -47,7 +47,7 @@ function getConsoleDB(): Database $database = new Database($dbAdapter, getCache()); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } diff --git a/app/worker.php b/app/worker.php index ea086fa43d..c15e6a2ab3 100644 --- a/app/worker.php +++ b/app/worker.php @@ -35,7 +35,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) ; $adapter = new Database($database, $cache); - $adapter->setNamespace('console'); + $adapter->setNamespace('_console'); return $adapter; }, ['cache', 'register']); diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 146500e743..548dc4871e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -223,7 +223,7 @@ abstract class Worker if (isset(self::$databases[$databaseName])) { $database = self::$databases[$databaseName]; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } @@ -237,7 +237,7 @@ abstract class Worker self::$databases[$databaseName] = $database; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } From ccd8c86e880781c604b98bb3773bea32cda61ae1 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:35:18 -0700 Subject: [PATCH 6/9] Skip updating _uid attribute and _permission index These updates make use of the PDO to manually execute a SQL statement, but the PDO is not available in the new migration platform task. So that we can move forward with the release, we will skip these updates and then figure it out in a subsequent release. --- src/Appwrite/Migration/Version/V19.php | 41 ++++++++++++++------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 74637c42ad..1841170c8b 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -516,29 +516,32 @@ class V19 extends Migration protected function alterPermissionIndex($collectionName): void { - try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - DROP INDEX `_permission`, - ADD INDEX `_permission` (`_permission`, `_type`, `_document`); - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } + // try { + // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; + // // $this->pdo->prepare(" + // // ALTER TABLE {$table} + // // DROP INDEX `_permission`, + // // ADD INDEX `_permission` (`_permission`, `_type`, `_document`); + // // ")->execute(); + // $this->projectDB->deleteIndex($collectionName, '_permission', ['_permission', '_type', '_document']); + // $this->projectDB->createIndex($collectionName, '_permission', Database::INDEX_KEY, ['_permission', '_type', '_document']) + // } catch (\Throwable $th) { + // Console::warning($th->getMessage()); + // } } protected function alterUidType($collectionName): void { - try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } + // try { + // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; + // // $this->pdo->prepare(" + // // ALTER TABLE {$table} + // // CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; + // // ")->execute(); + // $this->projectDB->updateAttribute($collectionName, '_uid', type: 'string', size: 255, required: true); + // } catch (\Throwable $th) { + // Console::warning($th->getMessage()); + // } } /** From dbba62fe245ecb93d7f6767572778b8ededbaf9d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 17 Aug 2023 22:14:55 -0400 Subject: [PATCH 7/9] Remove permission index and _uid column type change migrations --- src/Appwrite/Migration/Version/V19.php | 45 +------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 1841170c8b..ee7914a8e1 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -29,9 +29,6 @@ class V19 extends Migration Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); - $this->alterPermissionIndex('_metadata'); - $this->alterUidType('_metadata'); - Console::info('Migrating Databases'); $this->migrateDatabases(); @@ -58,14 +55,10 @@ class V19 extends Migration $databaseTable = "database_{$database->getInternalId()}"; - $this->alterPermissionIndex($databaseTable); - $this->alterUidType($databaseTable); foreach ($this->documentsIterator($databaseTable) as $collection) { $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}"; Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})"); - $this->alterPermissionIndex($collectionTable); - $this->alterUidType($collectionTable); } } } @@ -409,10 +402,6 @@ class V19 extends Migration default: break; } - if (!in_array($id, ['files', 'collections'])) { - $this->alterPermissionIndex($id); - $this->alterUidType($id); - } usleep(50000); } @@ -505,7 +494,7 @@ class V19 extends Migration break; case 'rules': // TODO: convert domains - + break; default: break; @@ -514,36 +503,6 @@ class V19 extends Migration return $document; } - protected function alterPermissionIndex($collectionName): void - { - // try { - // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; - // // $this->pdo->prepare(" - // // ALTER TABLE {$table} - // // DROP INDEX `_permission`, - // // ADD INDEX `_permission` (`_permission`, `_type`, `_document`); - // // ")->execute(); - // $this->projectDB->deleteIndex($collectionName, '_permission', ['_permission', '_type', '_document']); - // $this->projectDB->createIndex($collectionName, '_permission', Database::INDEX_KEY, ['_permission', '_type', '_document']) - // } catch (\Throwable $th) { - // Console::warning($th->getMessage()); - // } - } - - protected function alterUidType($collectionName): void - { - // try { - // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; - // // $this->pdo->prepare(" - // // ALTER TABLE {$table} - // // CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; - // // ")->execute(); - // $this->projectDB->updateAttribute($collectionName, '_uid', type: 'string', size: 255, required: true); - // } catch (\Throwable $th) { - // Console::warning($th->getMessage()); - // } - } - /** * Migrating all Bucket tables. * @@ -556,8 +515,6 @@ class V19 extends Migration foreach ($this->documentsIterator('buckets') as $bucket) { $id = "bucket_{$bucket->getInternalId()}"; Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); - $this->alterPermissionIndex($id); - $this->alterUidType($id); try { $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); From 2d53c73e7a2a73c36eca416a4811709e22fa7851 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 17 Aug 2023 22:15:33 -0400 Subject: [PATCH 8/9] Remove extra line --- src/Appwrite/Migration/Version/V19.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index ee7914a8e1..f015b8470e 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -55,7 +55,6 @@ class V19 extends Migration $databaseTable = "database_{$database->getInternalId()}"; - foreach ($this->documentsIterator($databaseTable) as $collection) { $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}"; Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})"); From d1ad1c0cf3320226efb197ad7943abc499a03880 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 22 Aug 2023 10:13:39 -0400 Subject: [PATCH 9/9] Fix lockfile --- composer.lock | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.lock b/composer.lock index e86397caec..5dbb741513 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "83427d18095b42c274d2bd6eb302e4aa", + "content-hash": "a05c1fe23b14b9f08345c9648540c304", "packages": [ { "name": "adhocore/jwt", @@ -3227,16 +3227,16 @@ }, { "name": "utopia-php/vcs", - "version": "dev-feat-clone-commit-hash", + "version": "0.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "5ed5d1685b3b714d6f20f7922ab4f5f5188313ee" + "reference": "ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/5ed5d1685b3b714d6f20f7922ab4f5f5188313ee", - "reference": "5ed5d1685b3b714d6f20f7922ab4f5f5188313ee", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf", + "reference": "ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf", "shasum": "" }, "require": { @@ -3270,9 +3270,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/feat-clone-commit-hash" + "source": "https://github.com/utopia-php/vcs/tree/0.2.0" }, - "time": "2023-08-21T09:02:54+00:00" + "time": "2023-08-21T10:59:48+00:00" }, { "name": "utopia-php/websocket", @@ -6069,18 +6069,9 @@ "time": "2023-07-26T07:16:09+00:00" } ], - "aliases": [ - { - "package": "utopia-php/vcs", - "version": "dev-feat-clone-commit-hash", - "alias": "0.1.99", - "alias_normalized": "0.1.99.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/vcs": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -6104,5 +6095,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" }