From 278fdf582321823fd235f59f4ce29fd62830573b Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 15 Jul 2024 12:17:10 +0300 Subject: [PATCH 1/4] clean migration project cache --- src/Appwrite/Platform/Tasks/Migrate.php | 45 ++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 21f83109ff..e11d7926ab 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -4,18 +4,26 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Migration\Migration; use Utopia\App; -use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; +use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Validator\Text; +use Utopia\System\System; +use Redis; class Migrate extends Action { + + /** + * @var Redis + */ + protected Redis $redis; + public static function getName(): string { return 'migrate'; @@ -27,23 +35,44 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') /** @TODO APP_VERSION_STABLE needs to be defined */ ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->inject('cache') ->inject('dbForConsole') ->inject('getProjectDB') ->inject('register') - ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register)); + ->callback(fn($version, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $dbForConsole, $getProjectDB, $register)); + + $this->redis = new Redis(); + $this->redis->connect( + System::getEnv('_APP_REDIS_HOST', null) , + System::getEnv('_APP_REDIS_PORT', 6379), + 3 , + null, + 10 + ); + } - private function clearProjectsCache(Cache $cache, Document $project) + private function clearProjectsCache(Document $project) { + try { - $cache->purge("cache-_{$project->getInternalId()}:*"); + do { + $iterator = null; + $pattern = "default-cache-_{$project->getInternalId()}:*"; + $keys = $this->redis->scan($iterator, $pattern, 1000); + if ($keys !== false) { + foreach ($keys as $key) { + $this->redis->del($key); + } + } + } while ($iterator > 0); + + } catch (\Throwable $th) { Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); } } - public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) + public function action(string $version, Database $dbForConsole, callable $getProjectDB, Registry $register) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -87,7 +116,7 @@ class Migrate extends Action continue; } - $this->clearProjectsCache($cache, $project); + $this->clearProjectsCache($project); try { // TODO: Iterate through all project DBs @@ -103,7 +132,7 @@ class Migrate extends Action throw $th; } - $this->clearProjectsCache($cache, $project); + $this->clearProjectsCache($project); } $sum = \count($projects); From ada1746466513e5b4abf6da6ceba098f889e7b42 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 15 Jul 2024 12:20:01 +0300 Subject: [PATCH 2/4] clean migration project cache --- src/Appwrite/Platform/Tasks/Migrate.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index e11d7926ab..58d3123726 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -10,7 +10,6 @@ use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; -use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Validator\Text; use Utopia\System\System; From 2ef358e8dc2765db882b664769f888a247e2441e Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 21 Jul 2024 09:27:12 +0300 Subject: [PATCH 3/4] linter --- src/Appwrite/Platform/Tasks/Migrate.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 58d3123726..336f8c9088 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Migration\Migration; +use Redis; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -11,13 +12,11 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Registry\Registry; -use Utopia\Validator\Text; use Utopia\System\System; -use Redis; +use Utopia\Validator\Text; class Migrate extends Action { - /** * @var Redis */ @@ -37,13 +36,13 @@ class Migrate extends Action ->inject('dbForConsole') ->inject('getProjectDB') ->inject('register') - ->callback(fn($version, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $dbForConsole, $getProjectDB, $register)); + ->callback(fn ($version, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $dbForConsole, $getProjectDB, $register)); $this->redis = new Redis(); $this->redis->connect( - System::getEnv('_APP_REDIS_HOST', null) , + System::getEnv('_APP_REDIS_HOST', null), System::getEnv('_APP_REDIS_PORT', 6379), - 3 , + 3, null, 10 ); @@ -71,7 +70,7 @@ class Migrate extends Action } } - public function action(string $version, Database $dbForConsole, callable $getProjectDB, Registry $register) + public function action(string $version, Database $dbForConsole, callable $getProjectDB, Registry $register) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { From 00fc34a2a42e9047b1d071cb4b3269df513e56d4 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 31 Jul 2024 13:23:51 +0300 Subject: [PATCH 4/4] get project db key by region --- src/Appwrite/Platform/Tasks/Migrate.php | 38 +++++++++++-------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 336f8c9088..4d921ff137 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -17,9 +17,6 @@ use Utopia\Validator\Text; class Migrate extends Action { - /** - * @var Redis - */ protected Redis $redis; public static function getName(): string @@ -38,20 +35,10 @@ class Migrate extends Action ->inject('register') ->callback(fn ($version, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $dbForConsole, $getProjectDB, $register)); - $this->redis = new Redis(); - $this->redis->connect( - System::getEnv('_APP_REDIS_HOST', null), - System::getEnv('_APP_REDIS_PORT', 6379), - 3, - null, - 10 - ); - } private function clearProjectsCache(Document $project) { - try { do { $iterator = null; @@ -64,24 +51,33 @@ class Migrate extends Action } } while ($iterator > 0); - } catch (\Throwable $th) { - Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); + Console::error('Failed to clear project ("'.$project->getId().'") cache with error: '.$th->getMessage()); } } public function action(string $version, Database $dbForConsole, callable $getProjectDB, Registry $register) { Authorization::disable(); - if (!array_key_exists($version, Migration::$versions)) { + if (! array_key_exists($version, Migration::$versions)) { Console::error("Version {$version} not found."); Console::exit(1); + return; } + $this->redis = new Redis(); + $this->redis->connect( + System::getEnv('_APP_REDIS_HOST', null), + System::getEnv('_APP_REDIS_PORT', 6379), + 3, + null, + 10 + ); + $app = new App('UTC'); - Console::success('Starting Data Migration to version ' . $version); + Console::success('Starting Data Migration to version '.$version); $console = $app->getResource('console'); @@ -101,11 +97,11 @@ class Migrate extends Action $totalProjects = $dbForConsole->count('projects') + 1; } - $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + $class = 'Appwrite\\Migration\\Version\\'.Migration::$versions[$version]; /** @var Migration $migration */ $migration = new $class(); - while (!empty($projects)) { + while (! empty($projects)) { foreach ($projects as $project) { /** * Skip user projects with id 'console' @@ -126,7 +122,7 @@ class Migrate extends Action ->setPDO($register->get('db', true)) ->execute(); } catch (\Throwable $th) { - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage()); throw $th; } @@ -139,7 +135,7 @@ class Migrate extends Action $offset = $offset + $limit; $count = $count + $sum; - Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); + Console::log('Migrated '.$count.'/'.$totalProjects.' projects...'); } Console::success('Data Migration Completed');