From b2381798ab2d8452ac6798a5cbfe853eabe42866 Mon Sep 17 00:00:00 2001 From: prateek banga Date: Mon, 20 Nov 2023 22:01:20 +0530 Subject: [PATCH 1/6] migrates enum attribute size to 255 --- app/init.php | 19 +++++++++++++++++++ src/Appwrite/Migration/Version/V19.php | 21 +++++++++++++++++++++ src/Appwrite/Platform/Tasks/Migrate.php | 8 ++++++-- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 4e8e8103c9..4cbf08cf2f 100644 --- a/app/init.php +++ b/app/init.php @@ -767,6 +767,25 @@ $register->set('pools', function () { return $group; }); +$register->set('db', function () { + // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbPort = App::getEnv('_APP_DB_PORT', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true, + )); + + return $pdo; +}); $register->set('influxdb', function () { // Register DB connection diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index b9ccf3c302..10a64d4b03 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -10,6 +10,7 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; +use Utopia\Database\Query; class V19 extends Migration { @@ -41,6 +42,11 @@ class V19 extends Migration Console::info('Migrating Buckets'); $this->migrateBuckets(); + if ($this->project->getId() !== 'console') { + Console::info('Migrating Enum Attribute Size'); + $this->migrateEnumAttributeSize(); + } + Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); @@ -640,6 +646,21 @@ class V19 extends Migration return $commands; } + private function migrateEnumAttributeSize(): void + { + \Co\run(function () { + foreach ($this->documentsIterator('attributes') as $attribute) { + go(function ($attribute) { + $attribute->setAttribute('size', Database::LENGTH_KEY); + $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); + $databaseInternalId = $attribute->getAttribute('databaseInternalId'); + $collectionInternalId = $attribute->getAttribute('collectionInternalId'); + $this->changeAttributeInternalType('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), 'varchar(255)'); + }, $attribute); + } + }); + } + /** * Fix run on each document * diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 38e5c8aa46..682b9b5559 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -11,6 +11,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -29,7 +30,8 @@ class Migrate extends Action ->inject('cache') ->inject('dbForConsole') ->inject('getProjectDB') - ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); + ->inject('register') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register)); } private function clearProjectsCache(Cache $cache, Document $project) @@ -41,7 +43,7 @@ class Migrate extends Action } } - public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -89,9 +91,11 @@ class Migrate extends Action try { // TODO: Iterate through all project DBs + /** @var Database $projectDB */ $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) + ->setPDO($register->get('db', true)) ->execute(); } catch (\Throwable $th) { Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); From da3798e52f80edbeec13f4add596087e7511c1ed Mon Sep 17 00:00:00 2001 From: prateek banga Date: Mon, 20 Nov 2023 23:31:34 +0530 Subject: [PATCH 2/6] review changes --- src/Appwrite/Migration/Version/V19.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 10a64d4b03..6f2cbfac49 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,17 +648,17 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - \Co\run(function () { - foreach ($this->documentsIterator('attributes') as $attribute) { - go(function ($attribute) { - $attribute->setAttribute('size', Database::LENGTH_KEY); - $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); - $databaseInternalId = $attribute->getAttribute('databaseInternalId'); - $collectionInternalId = $attribute->getAttribute('collectionInternalId'); - $this->changeAttributeInternalType('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), 'varchar(255)'); - }, $attribute); + foreach ($this->documentsIterator('attributes') as $attribute) { + if ($attribute->getAttribute('format') !== 'enum' || $attribute->getAttribute('size') >= Database::LENGTH_KEY) { + return; } - }); + + $attribute->setAttribute('size', Database::LENGTH_KEY); + $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); + $databaseInternalId = $attribute->getAttribute('databaseInternalId'); + $collectionInternalId = $attribute->getAttribute('collectionInternalId'); + $this->projectDB->updateAttribute('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), size: 255); + } } /** From 7e625ba653c0b40fb637f3015af58979aca256ee Mon Sep 17 00:00:00 2001 From: prateek banga Date: Tue, 21 Nov 2023 11:48:27 +0530 Subject: [PATCH 3/6] adds queries support in documentsIterator --- src/Appwrite/Migration/Migration.php | 12 +++++++++--- src/Appwrite/Migration/Version/V19.php | 6 +----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0a2dd2f186..7f2a362e7d 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -195,17 +195,23 @@ abstract class Migration * @return iterable * @throws \Exception */ - public function documentsIterator(string $collectionId): iterable + public function documentsIterator(string $collectionId, $queries = []): iterable { $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collectionId); + $queries[] = Query::limit($this->limit); do { - $queries = [Query::limit($this->limit)]; if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); + $lastQuery = $queries[\count($queries) - 1]; + if ($lastQuery->getMethod() === 'cursorAfter') { + $queries[\count($queries) - 1] = Query::cursorAfter($nextDocument); + } else { + $queries[] = Query::cursorAfter($nextDocument); + } } + $documents = $this->projectDB->find($collectionId, $queries); $count = count($documents); $sum += $count; diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 6f2cbfac49..b8c2c54650 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,11 +648,7 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - foreach ($this->documentsIterator('attributes') as $attribute) { - if ($attribute->getAttribute('format') !== 'enum' || $attribute->getAttribute('size') >= Database::LENGTH_KEY) { - return; - } - + foreach ($this->documentsIterator('attributes', [Query::equal('format', ['enum']), Query::lessThan('size', Database::LENGTH_KEY)]) as $attribute) { $attribute->setAttribute('size', Database::LENGTH_KEY); $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); $databaseInternalId = $attribute->getAttribute('databaseInternalId'); From 022f0a663edd0b870cf677faf2a10041adc6a5e5 Mon Sep 17 00:00:00 2001 From: prateek banga Date: Tue, 21 Nov 2023 12:23:36 +0530 Subject: [PATCH 4/6] changes array search for cursorAfter query --- src/Appwrite/Migration/Migration.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 7f2a362e7d..9ee4d9236a 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -204,9 +204,10 @@ abstract class Migration do { if ($nextDocument !== null) { - $lastQuery = $queries[\count($queries) - 1]; - if ($lastQuery->getMethod() === 'cursorAfter') { - $queries[\count($queries) - 1] = Query::cursorAfter($nextDocument); + $cursorQueryIndex = \array_search('cursorAfter', \array_map(fn (Query $query) => $query->getMethod(), $queries)); + + if ($cursorQueryIndex !== false) { + $queries[$cursorQueryIndex] = Query::cursorAfter($nextDocument); } else { $queries[] = Query::cursorAfter($nextDocument); } From aa6a1a2e31feff9a7702f5456c1869a4e8689ad5 Mon Sep 17 00:00:00 2001 From: Prateek Banga <30731059+fanatic75@users.noreply.github.com> Date: Tue, 21 Nov 2023 12:24:08 +0530 Subject: [PATCH 5/6] Update src/Appwrite/Migration/Version/V19.php Co-authored-by: Jake Barnby --- src/Appwrite/Migration/Version/V19.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index b8c2c54650..2131af4ff8 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,7 +648,10 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - foreach ($this->documentsIterator('attributes', [Query::equal('format', ['enum']), Query::lessThan('size', Database::LENGTH_KEY)]) as $attribute) { + foreach ($this->documentsIterator('attributes', [ + Query::equal('format', ['enum']), + Query::lessThan('size', Database::LENGTH_KEY) + ]) as $attribute) { $attribute->setAttribute('size', Database::LENGTH_KEY); $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); $databaseInternalId = $attribute->getAttribute('databaseInternalId'); From 1720acb2b40d593ab2fb3642257605c39ba059fd Mon Sep 17 00:00:00 2001 From: prateek banga Date: Tue, 21 Nov 2023 12:32:30 +0530 Subject: [PATCH 6/6] lint fix --- src/Appwrite/Migration/Version/V19.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 2131af4ff8..46a0a073a6 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,10 +648,12 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - foreach ($this->documentsIterator('attributes', [ - Query::equal('format', ['enum']), + foreach ( + $this->documentsIterator('attributes', [ + Query::equal('format', ['enum']), Query::lessThan('size', Database::LENGTH_KEY) - ]) as $attribute) { + ]) as $attribute + ) { $attribute->setAttribute('size', Database::LENGTH_KEY); $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); $databaseInternalId = $attribute->getAttribute('databaseInternalId');