From 22780b87dcaa8f7b1540fcaeebd99815827c0495 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:34:44 -0700 Subject: [PATCH 01/22] Bump appwrite version to 1.4.0 --- README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README-CN.md b/README-CN.md index b634458547..b4ce78b1f3 100644 --- a/README-CN.md +++ b/README-CN.md @@ -66,7 +66,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` ### Windows @@ -78,7 +78,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` #### PowerShell @@ -88,7 +88,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 2b82d49eb0..2fb6fb1351 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` ### Windows @@ -87,7 +87,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` #### PowerShell @@ -97,7 +97,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.3.8 + appwrite/appwrite:1.4.0 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 646a9d1cce..228fc4809e 100644 --- a/app/init.php +++ b/app/init.php @@ -107,8 +107,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 506; -const APP_VERSION_STABLE = '1.3.8'; +const APP_CACHE_BUSTER = 507; +const APP_VERSION_STABLE = '1.4.0'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; From 586ad1984cf9d7d1470de415fbebf28e31c5aad4 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:36:10 -0700 Subject: [PATCH 02/22] Update 1.4 migration --- src/Appwrite/Migration/Version/V19.php | 419 ++++++++++++++++++++++++- 1 file changed, 409 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 39522aff0b..44d0b308d5 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -2,8 +2,6 @@ namespace Appwrite\Migration\Version; -use Appwrite\Auth\Auth; -use Utopia\Config\Config; use Appwrite\Migration\Migration; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -76,24 +74,349 @@ class V19 extends Migration */ private function migrateCollections(): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + $collectionType = match ($internalProjectId) { + 'console' => 'console', + default => 'projects', + }; + $collections = $this->collections[$collectionType]; + foreach ($collections as $collection) { $id = $collection['$id']; + + if ($id === 'schedules' && $internalProjectId === 'console') { + continue; + } + Console::log("Migrating Collection \"{$id}\""); $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); switch ($id) { - case 'projects': + case '_metadata': + $this->createCollection('identities'); + $this->createCollection('migrations'); + $this->createCollection('statsLogger'); // TODO: should we do this now? + break; + case 'attributes': + case 'indexes': try { - /** - * Create 'passwordHistory' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'smtp'); - $this->createAttributeFromCollection($this->projectDB, $id, 'templates'); + $this->projectDB->updateAttribute($id, 'databaseInternalId', required: true); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'SMTP and Templates' from {$id}: {$th->getMessage()}"); + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'collectionInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'error'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'error' from {$id}: {$th->getMessage()}"); + } + break; + case 'builds': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + $this->projectDB->deleteCachedCollection($id); + + // TODO: stderr and stdout => logs + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'certificates': + try { + $this->projectDB->renameAttribute($id, 'log', 'logs'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'logs', size: 1000000); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + break; + case 'databases': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'enabled'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'enabled' from {$id}: {$th->getMessage()}"); + } + break; + case 'deployments': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'buildInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'buildInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'executions': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stderr', 'errors'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stdout', 'logs'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'logs' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'statusCode', 'responseStatusCode'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'responseStatusCode' from {$id}: {$th->getMessage()}"); + } + break; + case 'functions': + $attributesToCreate = [ + 'live', + 'installationId', + 'installationInternalId', + 'providerRepositoryId', + 'repositoryId', + 'repositoryInternalId', + 'providerBranch', + 'providerRootDirectory', + 'providerSilentMode', + 'logging', + 'deploymentInternalId', + 'scheduleInternalId', + 'scheduleId', + 'version', + 'entrypoint', + 'commands', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + '_key_installationId', + '_key_installationInternalId', + '_key_providerRepositoryId', + '_key_repositoryId', + '_key_repositoryInternalId', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + break; + case 'memberships': + try { + $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + // Intentional fall through to update memberships.userInternalId + case 'sessions': + case 'tokens': + try { + $this->projectDB->updateAttribute($id, 'userInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'domains': + case 'keys': + case 'platforms': + case 'webhooks': + try { + $this->projectDB->updateAttribute($id, 'projectInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'projects': + $attributesToCreate = [ + 'database', + 'enabled', + 'smtp', + 'templates', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + } + } + + // TODO: delete domains? + break; + case 'schedules': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'stats': + // TODO: should we do this now? + try { + $this->projectDB->updateAttribute($id, 'value', signed: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteAttribute($id, 'type'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteIndex($id, '_key_metric_period_time'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'teams': + try { + $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'users': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'labels'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'labels' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'accessedAt' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'search', filters: ['userSearch']); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'search' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_accessedAt'); + } catch (\Throwable $th) { + Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}"); + } + break; + case 'variables': + try { + $this->projectDB->deleteIndex($id, '_key_function'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteIndex($id, '_key_uniqueKey'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceType'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceType' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionInternalId', 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionId', 'resourceId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + $indexesToCreate = [ + '_key_resourceInternalId', + '_key_resourceId', + '_key_resourceType', + '_key_uniqueKey', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } } break; default: @@ -117,6 +440,69 @@ class V19 extends Migration protected function fixDocument(Document $document): Document { switch ($document->getCollection()) { + case '_metadata': + // TODO: function schedules + + // TODO: migrate statsLogger? + break; + case 'attributes': + case 'indexes': + $status = $document->getAttribute('status', ''); + if ($status === 'failed') { + $document->setAttribute('error', 'Unknown problem'); + } + break; + case 'builds': + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + break; + case 'collections': + case 'databases': + $document->setAttribute('enabled', true); + break; + case 'deployments': + $resourceId = $document->getAttribute('resourceId'); + $function = $this->projectDB->getDocument('functions', $resourceId); + $document->setAttribute('resourceInternalId', $function->getInternalId()); + + $buildId = $document->getAttribute('buildId'); + $build = $this->projectDB->getDocument('builds', $buildId); + $document->setAttribute('buildInternalId', $build->getInternalId()); + + $document->setAttribute('type', 'manual'); + break; + case 'executions': + $functionId = $document->getAttribute('functionId'); + $function = $this->projectDB->getDocument('functions', $functionId); + $document->setAttribute('functionInternalId', $function->getInternalId()); + + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + break; + case 'functions': + $document->setAttribute('live', true); + $document->setAttribute('logging', true); + $document->setAttribute('version', 'v2'); + $deploymentId = $document->getAttribute('deployment'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + + $schedule = $this->consoleDB->createDocument('schedules', new Document([ + 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'resourceType' => 'function', + 'resourceId' => $document->getId(), + 'resourceInternalId' => $document->getInternalId(), + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $this->project->getId(), + 'schedule' => $document->getAttribute('schedule'), + 'active' => !empty($document->getAttribute('schedule')) && !empty($document->getAttribute('deployment')), + ])); + + $document->setAttribute('scheduleId', $schedule->getId()); + $document->setAttribute('scheduleInternalId', $schedule->getInternalId()); + break; case 'projects': /** * Bump version number. @@ -126,6 +512,12 @@ class V19 extends Migration $document->setAttribute('smtp', []); $document->setAttribute('templates', []); + break; + case 'rules': + // TODO: convert domains + + break; + default: break; } @@ -173,6 +565,13 @@ class V19 extends Migration Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); $this->alterPermissionIndex($id); $this->alterUidType($id); + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); + } } } } From 436600547c1772f718cf4dc1546759a6c456cb8d Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:37:16 -0700 Subject: [PATCH 03/22] Fix Migration task after it was converted to a Platform task --- src/Appwrite/Migration/Migration.php | 41 ++++++++++++++++++++----- src/Appwrite/Migration/Version/V19.php | 36 ++++++++-------------- src/Appwrite/Platform/Tasks/Migrate.php | 34 +++++++++----------- 3 files changed, 61 insertions(+), 50 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 9ea3443091..9c6fbe6b39 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -77,7 +77,11 @@ abstract class Migration Authorization::disable(); Authorization::setDefaultStatus(false); - $this->collections = array_merge([ + $this->collections = Config::getParam('collections', []); + + $projectCollections = $this->collections['projects']; + + $this->collections['projects'] = array_merge([ '_metadata' => [ '$id' => ID::custom('_metadata'), '$collection' => Database::METADATA @@ -90,7 +94,7 @@ abstract class Migration '$id' => ID::custom('abuse'), '$collection' => Database::METADATA ] - ], Config::getParam('collections', [])); + ], $projectCollections); } /** @@ -131,7 +135,14 @@ abstract class Migration */ public function forEachDocument(callable $callback): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + + $collections = match ($internalProjectId) { + 'console' => $this->collections['console'], + default => $this->collections['projects'], + }; + + foreach ($collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { continue; } @@ -237,10 +248,15 @@ abstract class Migration { $name ??= $id; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) { $attributes = []; $indexes = []; - $collection = $this->collections[$id]; + $collection = $this->collections[$collectionType][$id]; foreach ($collection['attributes'] as $attribute) { $attributes[] = new Document([ @@ -286,9 +302,16 @@ abstract class Migration public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$from] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if ($from === 'files') { + $collectionType = 'buckets'; + } + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { - throw new Exception("Collection {$collectionId} not found"); + throw new Exception("Collection {$from} not found"); } $attributes = $collection['attributes']; @@ -332,7 +355,11 @@ abstract class Migration public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$collectionId] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 44d0b308d5..74637c42ad 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -3,8 +3,11 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; +use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; class V19 extends Migration @@ -90,7 +93,7 @@ class V19 extends Migration Console::log("Migrating Collection \"{$id}\""); - $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + $this->projectDB->setNamespace("_$internalProjectId"); switch ($id) { case '_metadata': @@ -294,28 +297,20 @@ class V19 extends Migration // TODO: delete domains? break; - case 'schedules': - try { - $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); - } - break; case 'stats': // TODO: should we do this now? try { $this->projectDB->updateAttribute($id, 'value', signed: true); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'value' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->deleteAttribute($id, 'type'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'type' from {$id}: {$th->getMessage()}"); } try { @@ -329,15 +324,7 @@ class V19 extends Migration $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); - } - break; - case 'teams': - try { - $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); } break; case 'users': @@ -441,8 +428,6 @@ class V19 extends Migration { switch ($document->getCollection()) { case '_metadata': - // TODO: function schedules - // TODO: migrate statsLogger? break; case 'attributes': @@ -488,6 +473,7 @@ class V19 extends Migration $deploymentId = $document->getAttribute('deployment'); $deployment = $this->projectDB->getDocument('deployments', $deploymentId); $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + $document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint')); $schedule = $this->consoleDB->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region @@ -509,6 +495,10 @@ class V19 extends Migration */ $document->setAttribute('version', '1.4.0'); + $databases = Config::getParam('pools-database', []); + $database = $databases[0]; + + $document->setAttribute('database', $database); $document->setAttribute('smtp', []); $document->setAttribute('templates', []); @@ -567,7 +557,7 @@ class V19 extends Migration $this->alterUidType($id); try { - $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 90b4234109..38e5c8aa46 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -7,10 +7,10 @@ use Utopia\CLI\Console; use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -26,20 +26,22 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') /** @TODO APP_VERSION_STABLE needs to be defined */ ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->inject('register') - ->callback(fn ($version, $register) => $this->action($version, $register)); + ->inject('cache') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); } - private function clearProjectsCache(Redis $redis, Document $project) + private function clearProjectsCache(Cache $cache, Document $project) { try { - $redis->del($redis->keys("cache-_{$project->getInternalId()}:*")); + $cache->purge("cache-_{$project->getInternalId()}:*"); } catch (\Throwable $th) { Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); } } - public function action(string $version, Registry $register) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -52,14 +54,6 @@ class Migrate extends Action Console::success('Starting Data Migration to version ' . $version); - $dbPool = $register->get('dbPool', true); - $redis = $register->get('cache', true); - - $cache = new Cache(new RedisCache($redis)); - - $dbForConsole = $dbPool->getDB('console', $cache); - $dbForConsole->setNamespace('_project_console'); - $console = $app->getResource('console'); $limit = 30; @@ -79,6 +73,7 @@ class Migrate extends Action } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + /** @var Migration $migration */ $migration = new $class(); while (!empty($projects)) { @@ -90,11 +85,11 @@ class Migrate extends Action continue; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); try { // TODO: Iterate through all project DBs - $projectDB = $dbPool->getDB($project->getId(), $cache); + $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) ->execute(); @@ -103,11 +98,11 @@ class Migrate extends Action throw $th; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); } $sum = \count($projects); - $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); + $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]); $offset = $offset + $limit; $count = $count + $sum; @@ -115,7 +110,6 @@ class Migrate extends Action Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); } - Swoole\Event::wait(); // Wait for Coroutines to finish Console::success('Data Migration Completed'); } } From 9f6b27f828d726131a971956622668c2626a30c8 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:32:32 -0700 Subject: [PATCH 04/22] Uncomment new db env vars --- app/config/variables.php | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 4187dc69bd..23af28f56d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -342,24 +342,24 @@ return [ 'question' => '', 'filter' => '' ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_PROJECT', -// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_CONSOLE', -// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ] + [ + 'name' => '_APP_CONNECTIONS_DB_PROJECT', + 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ] ], ], [ From e8f522975689fbcb5bd0bb924585d8c7ce2e0f16 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:34:54 -0700 Subject: [PATCH 05/22] Fix console db namespace --- app/cli.php | 2 +- app/init.php | 2 +- app/realtime.php | 2 +- app/worker.php | 2 +- src/Appwrite/Resque/Worker.php | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index d1dd885775..b410ab2ac7 100644 --- a/app/cli.php +++ b/app/cli.php @@ -58,7 +58,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { ->getResource(); $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); + $dbForConsole->setNamespace('_console'); // Ensure tables exist $collections = Config::getParam('collections', [])['console']; diff --git a/app/init.php b/app/init.php index 228fc4809e..3aadab7c0d 100644 --- a/app/init.php +++ b/app/init.php @@ -1090,7 +1090,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $database = new Database($dbAdapter, $cache); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; }, ['pools', 'cache']); diff --git a/app/realtime.php b/app/realtime.php index 772eee49d6..25b0532b42 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -47,7 +47,7 @@ function getConsoleDB(): Database $database = new Database($dbAdapter, getCache()); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } diff --git a/app/worker.php b/app/worker.php index ea086fa43d..c15e6a2ab3 100644 --- a/app/worker.php +++ b/app/worker.php @@ -35,7 +35,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) ; $adapter = new Database($database, $cache); - $adapter->setNamespace('console'); + $adapter->setNamespace('_console'); return $adapter; }, ['cache', 'register']); diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 146500e743..548dc4871e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -223,7 +223,7 @@ abstract class Worker if (isset(self::$databases[$databaseName])) { $database = self::$databases[$databaseName]; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } @@ -237,7 +237,7 @@ abstract class Worker self::$databases[$databaseName] = $database; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } From ccd8c86e880781c604b98bb3773bea32cda61ae1 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:35:18 -0700 Subject: [PATCH 06/22] Skip updating _uid attribute and _permission index These updates make use of the PDO to manually execute a SQL statement, but the PDO is not available in the new migration platform task. So that we can move forward with the release, we will skip these updates and then figure it out in a subsequent release. --- src/Appwrite/Migration/Version/V19.php | 41 ++++++++++++++------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 74637c42ad..1841170c8b 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -516,29 +516,32 @@ class V19 extends Migration protected function alterPermissionIndex($collectionName): void { - try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - DROP INDEX `_permission`, - ADD INDEX `_permission` (`_permission`, `_type`, `_document`); - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } + // try { + // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; + // // $this->pdo->prepare(" + // // ALTER TABLE {$table} + // // DROP INDEX `_permission`, + // // ADD INDEX `_permission` (`_permission`, `_type`, `_document`); + // // ")->execute(); + // $this->projectDB->deleteIndex($collectionName, '_permission', ['_permission', '_type', '_document']); + // $this->projectDB->createIndex($collectionName, '_permission', Database::INDEX_KEY, ['_permission', '_type', '_document']) + // } catch (\Throwable $th) { + // Console::warning($th->getMessage()); + // } } protected function alterUidType($collectionName): void { - try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } + // try { + // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; + // // $this->pdo->prepare(" + // // ALTER TABLE {$table} + // // CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; + // // ")->execute(); + // $this->projectDB->updateAttribute($collectionName, '_uid', type: 'string', size: 255, required: true); + // } catch (\Throwable $th) { + // Console::warning($th->getMessage()); + // } } /** From dbba62fe245ecb93d7f6767572778b8ededbaf9d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 17 Aug 2023 22:14:55 -0400 Subject: [PATCH 07/22] Remove permission index and _uid column type change migrations --- src/Appwrite/Migration/Version/V19.php | 45 +------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 1841170c8b..ee7914a8e1 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -29,9 +29,6 @@ class V19 extends Migration Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); - $this->alterPermissionIndex('_metadata'); - $this->alterUidType('_metadata'); - Console::info('Migrating Databases'); $this->migrateDatabases(); @@ -58,14 +55,10 @@ class V19 extends Migration $databaseTable = "database_{$database->getInternalId()}"; - $this->alterPermissionIndex($databaseTable); - $this->alterUidType($databaseTable); foreach ($this->documentsIterator($databaseTable) as $collection) { $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}"; Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})"); - $this->alterPermissionIndex($collectionTable); - $this->alterUidType($collectionTable); } } } @@ -409,10 +402,6 @@ class V19 extends Migration default: break; } - if (!in_array($id, ['files', 'collections'])) { - $this->alterPermissionIndex($id); - $this->alterUidType($id); - } usleep(50000); } @@ -505,7 +494,7 @@ class V19 extends Migration break; case 'rules': // TODO: convert domains - + break; default: break; @@ -514,36 +503,6 @@ class V19 extends Migration return $document; } - protected function alterPermissionIndex($collectionName): void - { - // try { - // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; - // // $this->pdo->prepare(" - // // ALTER TABLE {$table} - // // DROP INDEX `_permission`, - // // ADD INDEX `_permission` (`_permission`, `_type`, `_document`); - // // ")->execute(); - // $this->projectDB->deleteIndex($collectionName, '_permission', ['_permission', '_type', '_document']); - // $this->projectDB->createIndex($collectionName, '_permission', Database::INDEX_KEY, ['_permission', '_type', '_document']) - // } catch (\Throwable $th) { - // Console::warning($th->getMessage()); - // } - } - - protected function alterUidType($collectionName): void - { - // try { - // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; - // // $this->pdo->prepare(" - // // ALTER TABLE {$table} - // // CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; - // // ")->execute(); - // $this->projectDB->updateAttribute($collectionName, '_uid', type: 'string', size: 255, required: true); - // } catch (\Throwable $th) { - // Console::warning($th->getMessage()); - // } - } - /** * Migrating all Bucket tables. * @@ -556,8 +515,6 @@ class V19 extends Migration foreach ($this->documentsIterator('buckets') as $bucket) { $id = "bucket_{$bucket->getInternalId()}"; Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); - $this->alterPermissionIndex($id); - $this->alterUidType($id); try { $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); From 2d53c73e7a2a73c36eca416a4811709e22fa7851 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 17 Aug 2023 22:15:33 -0400 Subject: [PATCH 08/22] Remove extra line --- src/Appwrite/Migration/Version/V19.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index ee7914a8e1..f015b8470e 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -55,7 +55,6 @@ class V19 extends Migration $databaseTable = "database_{$database->getInternalId()}"; - foreach ($this->documentsIterator($databaseTable) as $collection) { $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}"; Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})"); From 6bb04ca95ea538766c89bf903111063277b18cd6 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:34:44 -0700 Subject: [PATCH 09/22] Bump appwrite version to 1.4.0 --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index d5d0ea9748..4d99fe2514 100644 --- a/app/init.php +++ b/app/init.php @@ -108,7 +108,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 506; +const APP_CACHE_BUSTER = 507; const APP_VERSION_STABLE = '1.4.0'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; From 280e066fd432b556f746860de54912f9f8ce21e1 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:36:10 -0700 Subject: [PATCH 10/22] Update 1.4 migration --- src/Appwrite/Migration/Version/V19.php | 565 ++++++++++++++++++++++++- 1 file changed, 555 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 39522aff0b..4dd892a13a 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -2,8 +2,6 @@ namespace Appwrite\Migration\Version; -use Appwrite\Auth\Auth; -use Utopia\Config\Config; use Appwrite\Migration\Migration; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -42,6 +40,22 @@ class V19 extends Migration Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); + + try { + $this->projectDB->deleteAttribute('projects', 'domains'); + $this->projectDB->deleteCachedCollection('projects'); + } catch (\Throwable $th) { + Console::warning("'domains' from projects: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteAttribute('functions', 'schedule'); + $this->projectDB->deleteCachedCollection('functions'); + } catch (\Throwable $th) { + Console::warning("'schedule' from functions: {$th->getMessage()}"); + } + + // TODO: delete builds stderr and stdout } /** @@ -76,24 +90,452 @@ class V19 extends Migration */ private function migrateCollections(): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + $collectionType = match ($internalProjectId) { + 'console' => 'console', + default => 'projects', + }; + $collections = $this->collections[$collectionType]; + foreach ($collections as $collection) { $id = $collection['$id']; + + if ($id === 'schedules' && $internalProjectId === 'console') { + continue; + } + Console::log("Migrating Collection \"{$id}\""); $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); switch ($id) { - case 'projects': + case '_metadata': + $this->createCollection('identities'); + $this->createCollection('migrations'); + $this->createCollection('statsLogger'); // TODO: should we do this now? + break; + case 'attributes': + case 'indexes': try { - /** - * Create 'passwordHistory' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'smtp'); - $this->createAttributeFromCollection($this->projectDB, $id, 'templates'); + $this->projectDB->updateAttribute($id, 'databaseInternalId', required: true); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'SMTP and Templates' from {$id}: {$th->getMessage()}"); + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'collectionInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'error'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'error' from {$id}: {$th->getMessage()}"); + } + break; + case 'buckets': + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_name', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + ...$indexesToDelete + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + break; + case 'builds': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'logs'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'logs' from {$id}: {$th->getMessage()}"); + } + break; + case 'certificates': + try { + $this->projectDB->renameAttribute($id, 'log', 'logs'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'logs', size: 1000000); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + break; + case 'databases': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'enabled'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'enabled' from {$id}: {$th->getMessage()}"); + } + break; + case 'deployments': + $attributesToCreate = [ + 'resourceInternalId', + 'buildInternalId', + 'type', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("$attribute from {$id}: {$th->getMessage()}"); + } + } + + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_entrypoint', + '_key_resource', + '_key_resource_type', + '_key_buildId', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + '_key_resource', + '_key_resource_type', + '_key_buildId', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + break; + case 'executions': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stderr', 'errors'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stdout', 'logs'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'logs' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'statusCode', 'responseStatusCode'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'responseStatusCode' from {$id}: {$th->getMessage()}"); + } + break; + case 'files': + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_name', + '_key_signature', + '_key_mimeType', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + ...$indexesToDelete + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + break; + case 'functions': + $attributesToCreate = [ + 'live', + 'installationId', + 'installationInternalId', + 'providerRepositoryId', + 'repositoryId', + 'repositoryInternalId', + 'providerBranch', + 'providerRootDirectory', + 'providerSilentMode', + 'logging', + 'deploymentInternalId', + 'scheduleInternalId', + 'scheduleId', + 'version', + 'entrypoint', + 'commands', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + } + } + + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_name', + '_key_runtime', + '_key_deployment', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + ...$indexesToDelete, + '_key_installationId', + '_key_installationInternalId', + '_key_providerRepositoryId', + '_key_repositoryId', + '_key_repositoryInternalId', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + break; + case 'memberships': + try { + $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + // Intentional fall through to update memberships.userInternalId + case 'sessions': + case 'tokens': + try { + $this->projectDB->updateAttribute($id, 'userInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'domains': + case 'keys': + case 'platforms': + case 'webhooks': + try { + $this->projectDB->updateAttribute($id, 'projectInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'projects': + $attributesToCreate = [ + 'database', + 'smtp', + 'templates', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + Console::warning($th->getTraceAsString()); + } + } + break; + case 'schedules': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'stats': + // TODO: should we do this now? + try { + $this->projectDB->updateAttribute($id, 'value', signed: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteAttribute($id, 'type'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteIndex($id, '_key_metric_period_time'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'teams': + try { + $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + break; + case 'users': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'labels'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'labels' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'accessedAt' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'search', filters: ['userSearch']); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'search' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_accessedAt'); + } catch (\Throwable $th) { + Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}"); + } + break; + case 'variables': + try { + $this->projectDB->deleteIndex($id, '_key_function'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteIndex($id, '_key_uniqueKey'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceType'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceType' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionInternalId', 'resourceInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionId', 'resourceId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + $indexesToCreate = [ + '_key_resourceInternalId', + '_key_resourceId', + '_key_resourceType', + '_key_uniqueKey', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } } break; default: @@ -117,6 +559,76 @@ class V19 extends Migration protected function fixDocument(Document $document): Document { switch ($document->getCollection()) { + case '_metadata': + // TODO: function schedules + + // TODO: migrate statsLogger? + break; + case 'attributes': + case 'indexes': + $status = $document->getAttribute('status', ''); + if ($status === 'failed') { + $document->setAttribute('error', 'Unknown problem'); + } + break; + case 'builds': + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + + // TODO: how to combine stderr & stdout > logs? + break; + case 'collections': + case 'databases': + $document->setAttribute('enabled', true); + break; + case 'deployments': + $resourceId = $document->getAttribute('resourceId'); + $function = $this->projectDB->getDocument('functions', $resourceId); + $document->setAttribute('resourceInternalId', $function->getInternalId()); + + $buildId = $document->getAttribute('buildId'); + if (!empty($buildId)) { + $build = $this->projectDB->getDocument('builds', $buildId); + $document->setAttribute('buildInternalId', $build->getInternalId()); + } + + $document->setAttribute('type', 'manual'); + break; + case 'executions': + $functionId = $document->getAttribute('functionId'); + $function = $this->projectDB->getDocument('functions', $functionId); + $document->setAttribute('functionInternalId', $function->getInternalId()); + + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + break; + case 'functions': + $document->setAttribute('live', true); + $document->setAttribute('logging', true); + $document->setAttribute('version', 'v2'); + $deploymentId = $document->getAttribute('deployment'); + if (!empty($deploymentId)) { + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + $document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint')); + } + + $schedule = $this->consoleDB->createDocument('schedules', new Document([ + 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'resourceType' => 'function', + 'resourceId' => $document->getId(), + 'resourceInternalId' => $document->getInternalId(), + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $this->project->getId(), + 'schedule' => $document->getAttribute('schedule'), + 'active' => !empty($document->getAttribute('schedule')) && !empty($document->getAttribute('deployment')), + ])); + + $document->setAttribute('scheduleId', $schedule->getId()); + $document->setAttribute('scheduleInternalId', $schedule->getInternalId()); + break; case 'projects': /** * Bump version number. @@ -126,6 +638,32 @@ class V19 extends Migration $document->setAttribute('smtp', []); $document->setAttribute('templates', []); + break; + case 'rules': + $status = 'created'; + if ($document->getAttribute('verification', false)) { + $status = 'verified'; + } + + $ruleDocument = new Document([ + 'projectId' => $this->project->getId(), + 'projectInternalId' => $this->project->getInternalId(), + 'domain' => $document->getAttribute('domain'), + 'resourceType' => 'api', + 'resourceInternalId' => '', + 'resourceId' => '', + 'status' => $status, + 'certificateId' => $document->getAttribute('certificateId'), + ]); + + try { + $this->consoleDB->createDocument('rules', $ruleDocument); + } catch (\Throwable $th) { + Console::warning("Error migrating domain {$document->getAttribute('domain')}: {$th->getMessage()}"); + } + + break; + default: break; } @@ -173,6 +711,13 @@ class V19 extends Migration Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); $this->alterPermissionIndex($id); $this->alterUidType($id); + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); + } } } } From f3fbb2fb564fadb838c2adfa866651d1cbb3f698 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Tue, 15 Aug 2023 17:37:16 -0700 Subject: [PATCH 11/22] Fix Migration task after it was converted to a Platform task --- src/Appwrite/Migration/Migration.php | 41 ++++++++++++++++++++----- src/Appwrite/Migration/Version/V19.php | 35 ++++++++------------- src/Appwrite/Platform/Tasks/Migrate.php | 34 +++++++++----------- 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 9ea3443091..9c6fbe6b39 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -77,7 +77,11 @@ abstract class Migration Authorization::disable(); Authorization::setDefaultStatus(false); - $this->collections = array_merge([ + $this->collections = Config::getParam('collections', []); + + $projectCollections = $this->collections['projects']; + + $this->collections['projects'] = array_merge([ '_metadata' => [ '$id' => ID::custom('_metadata'), '$collection' => Database::METADATA @@ -90,7 +94,7 @@ abstract class Migration '$id' => ID::custom('abuse'), '$collection' => Database::METADATA ] - ], Config::getParam('collections', [])); + ], $projectCollections); } /** @@ -131,7 +135,14 @@ abstract class Migration */ public function forEachDocument(callable $callback): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + + $collections = match ($internalProjectId) { + 'console' => $this->collections['console'], + default => $this->collections['projects'], + }; + + foreach ($collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { continue; } @@ -237,10 +248,15 @@ abstract class Migration { $name ??= $id; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) { $attributes = []; $indexes = []; - $collection = $this->collections[$id]; + $collection = $this->collections[$collectionType][$id]; foreach ($collection['attributes'] as $attribute) { $attributes[] = new Document([ @@ -286,9 +302,16 @@ abstract class Migration public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$from] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if ($from === 'files') { + $collectionType = 'buckets'; + } + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { - throw new Exception("Collection {$collectionId} not found"); + throw new Exception("Collection {$from} not found"); } $attributes = $collection['attributes']; @@ -332,7 +355,11 @@ abstract class Migration public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$collectionId] ?? null; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 4dd892a13a..275ffb7286 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -3,8 +3,11 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; +use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; class V19 extends Migration @@ -106,7 +109,7 @@ class V19 extends Migration Console::log("Migrating Collection \"{$id}\""); - $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + $this->projectDB->setNamespace("_$internalProjectId"); switch ($id) { case '_metadata': @@ -413,28 +416,20 @@ class V19 extends Migration } } break; - case 'schedules': - try { - $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); - } - break; case 'stats': // TODO: should we do this now? try { $this->projectDB->updateAttribute($id, 'value', signed: true); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'value' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->deleteAttribute($id, 'type'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'type' from {$id}: {$th->getMessage()}"); } try { @@ -448,15 +443,7 @@ class V19 extends Migration $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); - } - break; - case 'teams': - try { - $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); } break; case 'users': @@ -560,8 +547,6 @@ class V19 extends Migration { switch ($document->getCollection()) { case '_metadata': - // TODO: function schedules - // TODO: migrate statsLogger? break; case 'attributes': @@ -635,6 +620,10 @@ class V19 extends Migration */ $document->setAttribute('version', '1.4.0'); + $databases = Config::getParam('pools-database', []); + $database = $databases[0]; + + $document->setAttribute('database', $database); $document->setAttribute('smtp', []); $document->setAttribute('templates', []); @@ -713,7 +702,7 @@ class V19 extends Migration $this->alterUidType($id); try { - $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 90b4234109..38e5c8aa46 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -7,10 +7,10 @@ use Utopia\CLI\Console; use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -26,20 +26,22 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') /** @TODO APP_VERSION_STABLE needs to be defined */ ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->inject('register') - ->callback(fn ($version, $register) => $this->action($version, $register)); + ->inject('cache') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); } - private function clearProjectsCache(Redis $redis, Document $project) + private function clearProjectsCache(Cache $cache, Document $project) { try { - $redis->del($redis->keys("cache-_{$project->getInternalId()}:*")); + $cache->purge("cache-_{$project->getInternalId()}:*"); } catch (\Throwable $th) { Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); } } - public function action(string $version, Registry $register) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -52,14 +54,6 @@ class Migrate extends Action Console::success('Starting Data Migration to version ' . $version); - $dbPool = $register->get('dbPool', true); - $redis = $register->get('cache', true); - - $cache = new Cache(new RedisCache($redis)); - - $dbForConsole = $dbPool->getDB('console', $cache); - $dbForConsole->setNamespace('_project_console'); - $console = $app->getResource('console'); $limit = 30; @@ -79,6 +73,7 @@ class Migrate extends Action } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + /** @var Migration $migration */ $migration = new $class(); while (!empty($projects)) { @@ -90,11 +85,11 @@ class Migrate extends Action continue; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); try { // TODO: Iterate through all project DBs - $projectDB = $dbPool->getDB($project->getId(), $cache); + $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) ->execute(); @@ -103,11 +98,11 @@ class Migrate extends Action throw $th; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); } $sum = \count($projects); - $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); + $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]); $offset = $offset + $limit; $count = $count + $sum; @@ -115,7 +110,6 @@ class Migrate extends Action Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); } - Swoole\Event::wait(); // Wait for Coroutines to finish Console::success('Data Migration Completed'); } } From 5d6029fe7dc8d54f7756c6747aee03097156cfe9 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:32:32 -0700 Subject: [PATCH 12/22] Uncomment new db env vars --- app/config/variables.php | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 4187dc69bd..23af28f56d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -342,24 +342,24 @@ return [ 'question' => '', 'filter' => '' ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_PROJECT', -// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_CONSOLE', -// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ] + [ + 'name' => '_APP_CONNECTIONS_DB_PROJECT', + 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ] ], ], [ From a13b7606a25adea82c5dc57569b7e0498226f2e5 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:34:54 -0700 Subject: [PATCH 13/22] Fix console db namespace --- app/cli.php | 2 +- app/init.php | 2 +- app/realtime.php | 2 +- app/worker.php | 2 +- src/Appwrite/Resque/Worker.php | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index 2d12d69adb..fd1b8aff15 100644 --- a/app/cli.php +++ b/app/cli.php @@ -58,7 +58,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { ->getResource(); $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); + $dbForConsole->setNamespace('_console'); // Ensure tables exist $collections = Config::getParam('collections', [])['console']; diff --git a/app/init.php b/app/init.php index 4d99fe2514..0e327cd468 100644 --- a/app/init.php +++ b/app/init.php @@ -1116,7 +1116,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $database = new Database($dbAdapter, $cache); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; }, ['pools', 'cache']); diff --git a/app/realtime.php b/app/realtime.php index 772eee49d6..25b0532b42 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -47,7 +47,7 @@ function getConsoleDB(): Database $database = new Database($dbAdapter, getCache()); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } diff --git a/app/worker.php b/app/worker.php index 75fc95e9ac..084569886f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -37,7 +37,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) ; $adapter = new Database($database, $cache); - $adapter->setNamespace('console'); + $adapter->setNamespace('_console'); return $adapter; }, ['cache', 'register']); diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 146500e743..548dc4871e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -223,7 +223,7 @@ abstract class Worker if (isset(self::$databases[$databaseName])) { $database = self::$databases[$databaseName]; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } @@ -237,7 +237,7 @@ abstract class Worker self::$databases[$databaseName] = $database; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } From 45be279b75e5e4acaadcc5257acb096147593ffe Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 16 Aug 2023 16:35:18 -0700 Subject: [PATCH 14/22] Skip updating _uid attribute and _permission index These updates make use of the PDO to manually execute a SQL statement, but the PDO is not available in the new migration platform task. So that we can move forward with the release, we will skip these updates and then figure it out in a subsequent release. --- src/Appwrite/Migration/Version/V19.php | 41 ++++++++++++++------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 275ffb7286..704af811dd 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -661,29 +661,32 @@ class V19 extends Migration protected function alterPermissionIndex($collectionName): void { - try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - DROP INDEX `_permission`, - ADD INDEX `_permission` (`_permission`, `_type`, `_document`); - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } + // try { + // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; + // // $this->pdo->prepare(" + // // ALTER TABLE {$table} + // // DROP INDEX `_permission`, + // // ADD INDEX `_permission` (`_permission`, `_type`, `_document`); + // // ")->execute(); + // $this->projectDB->deleteIndex($collectionName, '_permission', ['_permission', '_type', '_document']); + // $this->projectDB->createIndex($collectionName, '_permission', Database::INDEX_KEY, ['_permission', '_type', '_document']) + // } catch (\Throwable $th) { + // Console::warning($th->getMessage()); + // } } protected function alterUidType($collectionName): void { - try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } + // try { + // // $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; + // // $this->pdo->prepare(" + // // ALTER TABLE {$table} + // // CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; + // // ")->execute(); + // $this->projectDB->updateAttribute($collectionName, '_uid', type: 'string', size: 255, required: true); + // } catch (\Throwable $th) { + // Console::warning($th->getMessage()); + // } } /** From ebe114617e8b50de708146087a7f799d5a564c7c Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Fri, 18 Aug 2023 13:58:16 -0700 Subject: [PATCH 15/22] Fix "Pool 'console' is empty" error by breaking from loop when ready --- app/cli.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/cli.php b/app/cli.php index fd1b8aff15..c8ebd089b1 100644 --- a/app/cli.php +++ b/app/cli.php @@ -74,7 +74,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { $pools->get('console')->reclaim(); sleep($sleep); } - } while ($attempts < $maxAttempts); + } while ($attempts < $maxAttempts && !$ready); if (!$ready) { throw new Exception("Console is not ready yet. Please try again later."); From 8ebfbf04b83964acfd7599ebedadcb429b901718 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Mon, 21 Aug 2023 15:38:00 -0700 Subject: [PATCH 16/22] Revert usage/stats migrations We won't releasing the usage changes this release because the migrations would be too big/complex. --- src/Appwrite/Migration/Version/V19.php | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 704af811dd..0568ce488b 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -115,7 +115,8 @@ class V19 extends Migration case '_metadata': $this->createCollection('identities'); $this->createCollection('migrations'); - $this->createCollection('statsLogger'); // TODO: should we do this now? + // Holding off on this until a future release + // $this->createCollection('statsLogger'); break; case 'attributes': case 'indexes': @@ -417,7 +418,6 @@ class V19 extends Migration } break; case 'stats': - // TODO: should we do this now? try { $this->projectDB->updateAttribute($id, 'value', signed: true); $this->projectDB->deleteCachedCollection($id); @@ -425,26 +425,27 @@ class V19 extends Migration Console::warning("'value' from {$id}: {$th->getMessage()}"); } - try { - $this->projectDB->deleteAttribute($id, 'type'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'type' from {$id}: {$th->getMessage()}"); - } + // Holding off on these until a future release + // try { + // $this->projectDB->deleteAttribute($id, 'type'); + // $this->projectDB->deleteCachedCollection($id); + // } catch (\Throwable $th) { + // Console::warning("'type' from {$id}: {$th->getMessage()}"); + // } - try { - $this->projectDB->deleteIndex($id, '_key_metric_period_time'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); - } + // try { + // $this->projectDB->deleteIndex($id, '_key_metric_period_time'); + // $this->projectDB->deleteCachedCollection($id); + // } catch (\Throwable $th) { + // Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + // } - try { - $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); - } + // try { + // $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); + // $this->projectDB->deleteCachedCollection($id); + // } catch (\Throwable $th) { + // Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + // } break; case 'users': try { @@ -547,7 +548,6 @@ class V19 extends Migration { switch ($document->getCollection()) { case '_metadata': - // TODO: migrate statsLogger? break; case 'attributes': case 'indexes': From 7e6eb59cdf82d0195c11206d414ddca6445df53f Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Mon, 21 Aug 2023 17:12:10 -0700 Subject: [PATCH 17/22] Additional updates from index changes --- src/Appwrite/Migration/Migration.php | 2 +- src/Appwrite/Migration/Version/V19.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 9c6fbe6b39..c569899918 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -369,7 +369,7 @@ abstract class Migration $indexKey = array_search($indexId, array_column($indexes, '$id')); if ($indexKey === false) { - throw new Exception("Attribute {$indexId} not found"); + throw new Exception("Index {$indexId} not found"); } $index = $indexes[$indexKey]; diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 0568ce488b..082c5c093c 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -487,7 +487,7 @@ class V19 extends Migration $this->projectDB->deleteIndex($id, '_key_uniqueKey'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + Console::warning("'_key_uniqueKey' from {$id}: {$th->getMessage()}"); } try { @@ -508,12 +508,12 @@ class V19 extends Migration $this->projectDB->renameAttribute($id, 'functionId', 'resourceId'); $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'resourceId' from {$id}: {$th->getMessage()}"); } $indexesToCreate = [ '_key_resourceInternalId', - '_key_resourceId', + '_key_resourceId_resourceType', '_key_resourceType', '_key_uniqueKey', ]; From d1ad1c0cf3320226efb197ad7943abc499a03880 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 22 Aug 2023 10:13:39 -0400 Subject: [PATCH 18/22] Fix lockfile --- composer.lock | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.lock b/composer.lock index e86397caec..5dbb741513 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "83427d18095b42c274d2bd6eb302e4aa", + "content-hash": "a05c1fe23b14b9f08345c9648540c304", "packages": [ { "name": "adhocore/jwt", @@ -3227,16 +3227,16 @@ }, { "name": "utopia-php/vcs", - "version": "dev-feat-clone-commit-hash", + "version": "0.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "5ed5d1685b3b714d6f20f7922ab4f5f5188313ee" + "reference": "ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/5ed5d1685b3b714d6f20f7922ab4f5f5188313ee", - "reference": "5ed5d1685b3b714d6f20f7922ab4f5f5188313ee", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf", + "reference": "ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf", "shasum": "" }, "require": { @@ -3270,9 +3270,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/feat-clone-commit-hash" + "source": "https://github.com/utopia-php/vcs/tree/0.2.0" }, - "time": "2023-08-21T09:02:54+00:00" + "time": "2023-08-21T10:59:48+00:00" }, { "name": "utopia-php/websocket", @@ -6069,18 +6069,9 @@ "time": "2023-07-26T07:16:09+00:00" } ], - "aliases": [ - { - "package": "utopia-php/vcs", - "version": "dev-feat-clone-commit-hash", - "alias": "0.1.99", - "alias_normalized": "0.1.99.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/vcs": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -6104,5 +6095,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From 6f775a2411b77327108e7e51e8dcb82de223a03b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 22 Aug 2023 19:14:23 -0400 Subject: [PATCH 19/22] Review updates --- src/Appwrite/Migration/Migration.php | 38 ++----- src/Appwrite/Migration/Version/V19.php | 150 +++++++++++-------------- tests/unit/Migration/MigrationTest.php | 88 --------------- 3 files changed, 78 insertions(+), 198 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index c569899918..b4d09b2ca8 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -159,12 +159,12 @@ abstract class Migration $old = $document->getArrayCopy(); $new = call_user_func($callback, $document); - if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) { + if (is_null($new) || $new->getArrayCopy() == $old) { return; } try { - $new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document); + $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document); } catch (\Throwable $th) { Console::error('Failed to update document: ' . $th->getMessage()); return; @@ -210,32 +210,6 @@ abstract class Migration } while (!is_null($nextDocument)); } - /** - * Checks 2 arrays for differences. - * - * @param array $array1 - * @param array $array2 - * @return bool - */ - public static function hasDifference(array $array1, array $array2): bool - { - foreach ($array1 as $key => $value) { - if (is_array($value)) { - if (!isset($array2[$key]) || !is_array($array2[$key])) { - return true; - } else { - if (self::hasDifference($value, $array2[$key])) { - return true; - } - } - } elseif (!array_key_exists($key, $array2) || $array2[$key] !== $value) { - return true; - } - } - - return false; - } - /** * Creates colletion from the config collection. * @@ -302,17 +276,22 @@ abstract class Migration public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void { $from ??= $collectionId; + $collectionType = match ($this->project->getInternalId()) { 'console' => 'console', default => 'projects', }; + if ($from === 'files') { $collectionType = 'buckets'; } + $collection = $this->collections[$collectionType][$from] ?? null; + if (is_null($collection)) { throw new Exception("Collection {$from} not found"); } + $attributes = $collection['attributes']; $attributeKey = array_search($attributeId, array_column($attributes, '$id')); @@ -355,15 +334,18 @@ abstract class Migration public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void { $from ??= $collectionId; + $collectionType = match ($this->project->getInternalId()) { 'console' => 'console', default => 'projects', }; + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); } + $indexes = $collection['indexes']; $indexKey = array_search($indexId, array_column($indexes, '$id')); diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index e3dcfacc81..8e727ddd9f 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -9,6 +9,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception; class V19 extends Migration { @@ -29,9 +30,6 @@ class V19 extends Migration Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); - Console::info('Migrating Databases'); - $this->migrateDatabases(); - Console::info('Migrating Collections'); $this->migrateCollections(); @@ -59,21 +57,23 @@ class V19 extends Migration } /** - * Migrate all Databases. + * Migrating all Bucket tables. * * @return void * @throws \Exception + * @throws \PDOException */ - private function migrateDatabases(): void + protected function migrateBuckets(): void { - foreach ($this->documentsIterator('databases') as $database) { - Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})"); + foreach ($this->documentsIterator('buckets') as $bucket) { + $id = "bucket_{$bucket->getInternalId()}"; + Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); - $databaseTable = "database_{$database->getInternalId()}"; - - foreach ($this->documentsIterator($databaseTable) as $collection) { - $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}"; - Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})"); + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); } } } @@ -82,6 +82,8 @@ class V19 extends Migration * Migrate all Collections. * * @return void + * @throws \Throwable + * @throws Exception */ private function migrateCollections(): void { @@ -94,7 +96,6 @@ class V19 extends Migration foreach ($collections as $collection) { $id = $collection['$id']; - if ($id === 'schedules' && $internalProjectId === 'console') { continue; } @@ -114,23 +115,23 @@ class V19 extends Migration case 'indexes': try { $this->projectDB->updateAttribute($id, 'databaseInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->updateAttribute($id, 'collectionInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); } try { $this->createAttributeFromCollection($this->projectDB, $id, 'error'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'error' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'buckets': // Recreate indexes so they're the right size @@ -140,7 +141,6 @@ class V19 extends Migration foreach ($indexesToDelete as $index) { try { $this->projectDB->deleteIndex($id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } @@ -149,52 +149,59 @@ class V19 extends Migration $indexesToCreate = [ ...$indexesToDelete ]; + foreach ($indexesToCreate as $index) { try { $this->createIndexFromCollection($this->projectDB, $id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } } + + $this->projectDB->deleteCachedCollection($id); + break; case 'builds': try { $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); } try { $this->createAttributeFromCollection($this->projectDB, $id, 'logs'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'logs' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'certificates': try { $this->projectDB->renameAttribute($id, 'log', 'logs'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'errors' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->updateAttribute($id, 'logs', size: 1000000); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'errors' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'databases': try { $this->createAttributeFromCollection($this->projectDB, $id, 'enabled'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'enabled' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'deployments': $attributesToCreate = [ @@ -205,7 +212,6 @@ class V19 extends Migration foreach ($attributesToCreate as $attribute) { try { $this->createAttributeFromCollection($this->projectDB, $id, $attribute); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("$attribute from {$id}: {$th->getMessage()}"); } @@ -221,7 +227,6 @@ class V19 extends Migration foreach ($indexesToDelete as $index) { try { $this->projectDB->deleteIndex($id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } @@ -235,47 +240,47 @@ class V19 extends Migration foreach ($indexesToCreate as $index) { try { $this->createIndexFromCollection($this->projectDB, $id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } } + + $this->projectDB->deleteCachedCollection($id); + break; case 'executions': try { $this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}"); } try { $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->renameAttribute($id, 'stderr', 'errors'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'errors' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->renameAttribute($id, 'stdout', 'logs'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'logs' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->renameAttribute($id, 'statusCode', 'responseStatusCode'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'responseStatusCode' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'files': // Recreate indexes so they're the right size @@ -287,23 +292,22 @@ class V19 extends Migration foreach ($indexesToDelete as $index) { try { $this->projectDB->deleteIndex($id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } } - $indexesToCreate = [ - ...$indexesToDelete - ]; + $indexesToCreate = $indexesToDelete; foreach ($indexesToCreate as $index) { try { $this->createIndexFromCollection($this->projectDB, $id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } } + + $this->projectDB->deleteCachedCollection($id); + break; case 'functions': $attributesToCreate = [ @@ -327,7 +331,6 @@ class V19 extends Migration foreach ($attributesToCreate as $attribute) { try { $this->createAttributeFromCollection($this->projectDB, $id, $attribute); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); } @@ -342,7 +345,6 @@ class V19 extends Migration foreach ($indexesToDelete as $index) { try { $this->projectDB->deleteIndex($id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } @@ -359,28 +361,34 @@ class V19 extends Migration foreach ($indexesToCreate as $index) { try { $this->createIndexFromCollection($this->projectDB, $id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } } + + $this->projectDB->deleteCachedCollection($id); + break; case 'memberships': try { $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + // Intentional fall through to update memberships.userInternalId case 'sessions': case 'tokens': try { $this->projectDB->updateAttribute($id, 'userInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'domains': case 'keys': @@ -388,10 +396,12 @@ class V19 extends Migration case 'webhooks': try { $this->projectDB->updateAttribute($id, 'projectInternalId', required: true); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'projects': $attributesToCreate = [ @@ -402,17 +412,18 @@ class V19 extends Migration foreach ($attributesToCreate as $attribute) { try { $this->createAttributeFromCollection($this->projectDB, $id, $attribute); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); Console::warning($th->getTraceAsString()); } } + + $this->projectDB->deleteCachedCollection($id); + break; case 'stats': try { $this->projectDB->updateAttribute($id, 'value', signed: true); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'value' from {$id}: {$th->getMessage()}"); } @@ -438,25 +449,25 @@ class V19 extends Migration // } catch (\Throwable $th) { // Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); // } + + $this->projectDB->deleteCachedCollection($id); + break; case 'users': try { $this->createAttributeFromCollection($this->projectDB, $id, 'labels'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'labels' from {$id}: {$th->getMessage()}"); } try { $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'accessedAt' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->updateAttribute($id, 'search', filters: ['userSearch']); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'search' from {$id}: {$th->getMessage()}"); } @@ -466,39 +477,37 @@ class V19 extends Migration } catch (\Throwable $th) { Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}"); } + + $this->projectDB->deleteCachedCollection($id); + break; case 'variables': try { $this->projectDB->deleteIndex($id, '_key_function'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->deleteIndex($id, '_key_uniqueKey'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'_key_uniqueKey' from {$id}: {$th->getMessage()}"); } try { $this->createAttributeFromCollection($this->projectDB, $id, 'resourceType'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'resourceType' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->renameAttribute($id, 'functionInternalId', 'resourceInternalId'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); } try { $this->projectDB->renameAttribute($id, 'functionId', 'resourceId'); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'resourceId' from {$id}: {$th->getMessage()}"); } @@ -512,11 +521,13 @@ class V19 extends Migration foreach ($indexesToCreate as $index) { try { $this->createIndexFromCollection($this->projectDB, $id, $index); - $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'$index' from {$id}: {$th->getMessage()}"); } } + + $this->projectDB->deleteCachedCollection($id); + break; default: break; @@ -535,8 +546,6 @@ class V19 extends Migration protected function fixDocument(Document $document): Document { switch ($document->getCollection()) { - case '_metadata': - break; case 'attributes': case 'indexes': $status = $document->getAttribute('status', ''); @@ -549,9 +558,10 @@ class V19 extends Migration $deployment = $this->projectDB->getDocument('deployments', $deploymentId); $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); - // TODO: how to combine stderr & stdout > logs? + $stdout = $document->getAttribute('stdout', ''); + $stderr = $document->getAttribute('stderr', ''); + $document->setAttribute('logs', $stdout . PHP_EOL . $stderr); break; - case 'collections': case 'databases': $document->setAttribute('enabled', true); break; @@ -582,6 +592,7 @@ class V19 extends Migration $document->setAttribute('logging', true); $document->setAttribute('version', 'v2'); $deploymentId = $document->getAttribute('deployment'); + if (!empty($deploymentId)) { $deployment = $this->projectDB->getDocument('deployments', $deploymentId); $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); @@ -603,9 +614,6 @@ class V19 extends Migration $document->setAttribute('scheduleInternalId', $schedule->getInternalId()); break; case 'projects': - /** - * Bump version number. - */ $document->setAttribute('version', '1.4.0'); $databases = Config::getParam('pools-database', []); @@ -646,26 +654,4 @@ class V19 extends Migration return $document; } - - /** - * Migrating all Bucket tables. - * - * @return void - * @throws \Exception - * @throws \PDOException - */ - protected function migrateBuckets(): void - { - foreach ($this->documentsIterator('buckets') as $bucket) { - $id = "bucket_{$bucket->getInternalId()}"; - Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); - - try { - $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); - } - } - } } diff --git a/tests/unit/Migration/MigrationTest.php b/tests/unit/Migration/MigrationTest.php index 13f9b9a3f5..536278d55b 100644 --- a/tests/unit/Migration/MigrationTest.php +++ b/tests/unit/Migration/MigrationTest.php @@ -47,92 +47,4 @@ abstract class MigrationTest extends TestCase $this->assertArrayHasKey(APP_VERSION_STABLE, Migration::$versions); } } - - public function testHasDifference(): void - { - $this->assertFalse(Migration::hasDifference([], [])); - $this->assertFalse(Migration::hasDifference([ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ] - ], [ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ] - ])); - $this->assertFalse(Migration::hasDifference([ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ] - ], [ - 'string' => 'abc', - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ], - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'bool' => true, - - ])); - $this->assertTrue(Migration::hasDifference([ - 'a' => true - ], [ - 'b' => true - ])); - $this->assertTrue(Migration::hasDifference([ - 'a' => 'true' - ], [ - 'a' => true - ])); - $this->assertTrue(Migration::hasDifference([ - 'a' => true - ], [ - 'a' => false - ])); - $this->assertTrue(Migration::hasDifference([ - 'nested' => [ - 'a' => true - ] - ], [ - 'nested' => [] - ])); - $this->assertTrue(Migration::hasDifference([ - 'assoc' => [ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'] - ] - ], [ - 'nested' => [ - 'a' => true, - 'int' => '123', - 'array' => ['a', 'b', 'c'] - ] - ])); - } } From b66734e49bd2ed786b8853a195b87aed88ee805b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 22 Aug 2023 19:19:46 -0400 Subject: [PATCH 20/22] Revert commented vars --- app/config/variables.php | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 23af28f56d..f69a7e411a 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -342,24 +342,24 @@ return [ 'question' => '', 'filter' => '' ], - [ - 'name' => '_APP_CONNECTIONS_DB_PROJECT', - 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', - 'introduction' => 'TBD', - 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - 'required' => true, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_CONNECTIONS_DB_CONSOLE', - 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', - 'introduction' => 'TBD', - 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - 'required' => true, - 'question' => '', - 'filter' => '' - ] +// [ +// 'name' => '_APP_CONNECTIONS_DB_PROJECT', +// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', +// 'introduction' => 'TBD', +// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', +// 'required' => true, +// 'question' => '', +// 'filter' => '' +// ], +// [ +// 'name' => '_APP_CONNECTIONS_DB_CONSOLE', +// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', +// 'introduction' => 'TBD', +// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', +// 'required' => true, +// 'question' => '', +// 'filter' => '' +// ] ], ], [ From 41faa7420f05ba7e3dc8b70e5a8c8564a964d3d1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 22 Aug 2023 19:25:07 -0400 Subject: [PATCH 21/22] Fix diff --- app/config/variables.php | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index f69a7e411a..4187dc69bd 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -342,24 +342,24 @@ return [ 'question' => '', 'filter' => '' ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_PROJECT', -// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ], -// [ -// 'name' => '_APP_CONNECTIONS_DB_CONSOLE', -// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', -// 'introduction' => 'TBD', -// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', -// 'required' => true, -// 'question' => '', -// 'filter' => '' -// ] +// [ +// 'name' => '_APP_CONNECTIONS_DB_PROJECT', +// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', +// 'introduction' => 'TBD', +// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', +// 'required' => true, +// 'question' => '', +// 'filter' => '' +// ], +// [ +// 'name' => '_APP_CONNECTIONS_DB_CONSOLE', +// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', +// 'introduction' => 'TBD', +// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', +// 'required' => true, +// 'question' => '', +// 'filter' => '' +// ] ], ], [ From e15a9ac2800618d4b91a6c0f2a18c6cd5d81a45e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 22 Aug 2023 19:36:46 -0400 Subject: [PATCH 22/22] Clean up attributes to be deleted --- src/Appwrite/Migration/Version/V19.php | 51 ++++++++++++++++++-------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 8e727ddd9f..921aacbded 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -39,21 +39,8 @@ class V19 extends Migration Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); - try { - $this->projectDB->deleteAttribute('projects', 'domains'); - $this->projectDB->deleteCachedCollection('projects'); - } catch (\Throwable $th) { - Console::warning("'domains' from projects: {$th->getMessage()}"); - } - - try { - $this->projectDB->deleteAttribute('functions', 'schedule'); - $this->projectDB->deleteCachedCollection('functions'); - } catch (\Throwable $th) { - Console::warning("'schedule' from functions: {$th->getMessage()}"); - } - - // TODO: delete builds stderr and stdout + Console::log('Cleaning Up Collections'); + $this->cleanCollections(); } /** @@ -124,6 +111,7 @@ class V19 extends Migration } catch (\Throwable $th) { Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); } + try { $this->createAttributeFromCollection($this->projectDB, $id, 'error'); } catch (\Throwable $th) { @@ -654,4 +642,37 @@ class V19 extends Migration return $document; } + + private function cleanCollections(): void + { + try { + $this->projectDB->deleteAttribute('projects', 'domains'); + } catch (\Throwable $th) { + Console::warning("'domains' from projects: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection('projects'); + + try { + $this->projectDB->deleteAttribute('functions', 'schedule'); + } catch (\Throwable $th) { + Console::warning("'schedule' from functions: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection('functions'); + + try { + $this->projectDB->deleteAttribute('builds', 'stderr'); + } catch (\Throwable $th) { + Console::warning("'stderr' from builds: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteAttribute('builds', 'stdout'); + } catch (\Throwable $th) { + Console::warning("'stdout' from builds: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection('builds'); + } }