From 907d968a5f728af1e8bba9606a7ba69395484377 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 16 May 2025 17:25:06 +1200 Subject: [PATCH] Remove redundant doc migration --- src/Appwrite/Migration/Version/V23.php | 89 +++---------- src/Appwrite/Platform/Tasks/Migrate.php | 163 +++++++++++------------- 2 files changed, 94 insertions(+), 158 deletions(-) diff --git a/src/Appwrite/Migration/Version/V23.php b/src/Appwrite/Migration/Version/V23.php index ab986cab55..39d8e1ff3f 100644 --- a/src/Appwrite/Migration/Version/V23.php +++ b/src/Appwrite/Migration/Version/V23.php @@ -404,89 +404,38 @@ class V23 extends Migration 17. Convert build's "path" to "buildPath" 18. Convert build's "logs" to "buildLogs" 19. Fill "totalSize" with "buildSize" plus "sourceSize" - --- Support for functions dual writing, if function's deploymentId is current deployment - 1. Fill function's "deploymentCreatedAt" with deployment's "$createdAt" - --- Support for functions dual writing, if this is most recent deployment - 2. Fill function's "latestDeploymentId" with deployment's "$id" - 3. Fill function's "latestDeploymentInternalId" with deployment's "$internalId" - 4. Fill function's "latestDeploymentCreatedAt" with deployment's "$createdAt" - 5. Fill function's "latestDeploymentStatus" with deployment's "$status" */ $document - ->setAttribute("buildCommands", $document->getAttribute("commands")) - ->setAttribute("sourcePath", $document->getAttribute("path")) - ->setAttribute("sourceSize", $document->getAttribute("size")) - ->setAttribute("sourceMetadata", $document->getAttribute("metadata")) - ->setAttribute("sourceChunksTotal", $document->getAttribute("chunksTotal")) - ->setAttribute("sourceChunksUploaded", $document->getAttribute("chunksUploaded")) - ; + ->setAttribute('buildCommands', $document->getAttribute('commands')) + ->setAttribute('sourcePath', $document->getAttribute('path')) + ->setAttribute('sourceSize', $document->getAttribute('size')) + ->setAttribute('sourceMetadata', $document->getAttribute('metadata')) + ->setAttribute('sourceChunksTotal', $document->getAttribute('chunksTotal')) + ->setAttribute('sourceChunksUploaded', $document->getAttribute('chunksUploaded')); $build = new Document(); - if (!empty($document->getAttribute('buildId'))) { - $build = Authorization::skip(fn () => $this->projectDB->getDocument('builds', $document->getAttribute('buildId'))); + $build = $this->dbForProject->getDocument('builds', $document->getAttribute('buildId')); } $document - ->setAttribute("buildStartedAt", $build->getAttribute("startTime", null)) - ->setAttribute("buildEndedAt", $build->getAttribute("endTime", null)) - ->setAttribute("buildDuration", $build->getAttribute("duration", 0)) - ->setAttribute("buildSize", $build->getAttribute("size", 0)) - ->setAttribute("status", $build->getAttribute("status", null)) - ->setAttribute("buildPath", $build->getAttribute("path", "")) - ->setAttribute("buildLogs", $build->getAttribute("logs", "")) - ; + ->setAttribute('buildStartedAt', $build->getAttribute('startTime')) + ->setAttribute('buildEndedAt', $build->getAttribute('endTime')) + ->setAttribute('buildDuration', $build->getAttribute('duration', 0)) + ->setAttribute('buildSize', $build->getAttribute('size', 0)) + ->setAttribute('status', $build->getAttribute('status')) + ->setAttribute('buildPath', $build->getAttribute('path', '')) + ->setAttribute('buildLogs', $build->getAttribute('logs', '')); - $totalSize = $document->getAttribute('buildSize', 0) + $document->getAttribute('sourceSize', 0); - $document->setAttribute("totalSize", $totalSize); + $totalSize = $document->getAttribute('buildSize', 0) + + $document->getAttribute('sourceSize', 0); - $function = Authorization::skip(fn () => $this->projectDB->getDocument('functions', $document->getAttribute('resourceId'))); - if (!$function->isEmpty()) { - $activeDeploymentId = $function->getAttribute('deployment', $function->getAttribute('deploymentId', '')); - if ($activeDeploymentId === $document->getId()) { - $function->setAttribute('deploymentCreatedAt', $document->getCreatedAt()); - - $function = Authorization::skip(fn () => $this->projectDB->updateDocument('functions', $function->getId(), $function)); - } else { - $latestDeployments = Authorization::skip(fn () => $this->projectDB->find('deployments', [ - Query::orderDesc(), - Query::limit(1), - Query::equal('resourceType', ['functions']), - Query::equal('resourceId', [$function->getId()]), - ])); - $latestDeployment = $latestDeployments[0] ? $latestDeployments[0] : new Document(); - - if (!$latestDeployment->isEmpty()) { - if ($latestDeployment->getId() === $document->getId()) { - $function - ->setAttribute("latestDeploymentId", $document->getId()) - ->setAttribute("latestDeploymentInternalId", $document->getInternalId()) - ->setAttribute("latestDeploymentCreatedAt", $document->getCreatedAt()) - ->setAttribute("latestDeploymentStatus", $document->getAttribute('status')) - ; - - $function = Authorization::skip(fn () => $this->projectDB->updateDocument('functions', $function->getId(), $function)); - } - } - } - } - break; - case 'builds': - /** - 1. Trigger deployments document update - */ - if (!empty($document->getAttribute('deploymentId'))) { - $deployment = Authorization::skip(fn () => $this->projectDB->getDocument('deployments', $document->getAttribute('deploymentId'))); - if (!$deployment->isEmpty()) { - $deployment = $this->fixDocument($deployment); - - Authorization::skip(fn () => $this->projectDB->updateDocument('deployments', $deployment->getId(), $deployment)); - } - } + $document->setAttribute('totalSize', $totalSize); break; + case 'migrations': /* - 1. Fill "options" with "[]" + 1. Fill "options" with "[]" */ $document->setAttribute('options', []); break; diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index e495ce1d3f..c2c19ec998 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Migration\Migration; use Redis; +use Swoole\Runtime; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -26,9 +27,10 @@ class Migrate extends Action public function __construct() { + Runtime::enableCoroutine(); + $this ->desc('Migrate Appwrite to new version') - /** @TODO APP_VERSION_STABLE needs to be defined */ ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) ->inject('dbForPlatform') ->inject('getProjectDB') @@ -36,12 +38,82 @@ class Migrate extends Action ->callback($this->action(...)); } - private function clearProjectsCache(Document $project) + /** + * @param string $version + * @param Database $dbForPlatform + * @param callable(Document): Database $getProjectDB + * @param Registry $register + * @return void + */ + public function action( + string $version, + Database $dbForPlatform, + callable $getProjectDB, + Registry $register, + ): void + { + Authorization::disable(); + + if (!\array_key_exists($version, Migration::$versions)) { + Console::error("Version {$version} not found."); + Console::exit(1); + return; + } + + \Co\run(function ( + string $version, + Database $dbForPlatform, + callable $getProjectDB, + Registry $register, + ) { + $this->redis = new Redis(); + $this->redis->connect( + System::getEnv('_APP_REDIS_HOST', ''), + System::getEnv('_APP_REDIS_PORT', 6379), + 3, + null, + 10 + ); + + Console::success('Starting Data Migration to version ' . $version); + + + $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + + /** @var Migration $migration */ + $migration = new $class(); + + $count = 0; + $total = $dbForPlatform->count('projects'); + + $dbForPlatform->foreach('projects', function (Document $project) use ($dbForPlatform, $getProjectDB, $register, $migration, &$count, $total) { + /** @var Database $dbForProject */ + $dbForProject = $getProjectDB($project); + $dbForProject->disableValidation(); + + try { + $migration + ->setProject($project, $dbForProject, $dbForPlatform) + ->setPDO($register->get('db', true)) + ->execute(); + } catch (\Throwable $th) { + Console::error('Failed to migrate project "' . $project->getId() . '" with error: ' . $th->getMessage()); + throw $th; + } + + Console::log('Migrated ' . $count++ . '/' . $total . ' projects...'); + }); + + Console::success('Migration completed'); + }, [$version, $dbForPlatform, $getProjectDB, $register]); + } + + private function clearProjectsCache(Document $project): void { try { $iterator = null; do { - $pattern = "default-cache-_{$project->getInternalId()}:*"; + $pattern = "default-cache-mariadb:_{$project->getInternalId()}:*"; $keys = $this->redis->scan($iterator, $pattern, 1000); if ($keys !== false) { foreach ($keys as $key) { @@ -53,89 +125,4 @@ class Migrate extends Action Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); } } - - public function action(string $version, Database $dbForPlatform, callable $getProjectDB, Registry $register) - { - Authorization::disable(); - if (!array_key_exists($version, Migration::$versions)) { - Console::error("Version {$version} not found."); - Console::exit(1); - - return; - } - - $this->redis = new Redis(); - $this->redis->connect( - System::getEnv('_APP_REDIS_HOST', null), - System::getEnv('_APP_REDIS_PORT', 6379), - 3, - null, - 10 - ); - - $app = new App('UTC'); - - Console::success('Starting Data Migration to version ' . $version); - - $console = $app->getResource('console'); - - $limit = 30; - $sum = 30; - $offset = 0; - /** - * @var \Utopia\Database\Document[] $projects - */ - $projects = [$console]; - $count = 0; - - try { - $totalProjects = $dbForPlatform->count('projects') + 1; - } catch (\Throwable $th) { - $dbForPlatform->setNamespace('_console'); - $totalProjects = $dbForPlatform->count('projects') + 1; - } - - $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; - /** @var Migration $migration */ - $migration = new $class(); - - while (!empty($projects)) { - foreach ($projects as $project) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console' && $project->getInternalId() !== 'console') { - continue; - } - - $this->clearProjectsCache($project); - - try { - // TODO: Iterate through all project DBs - /** @var Database $projectDB */ - $projectDB = $getProjectDB($project); - $projectDB->disableValidation(); - $migration - ->setProject($project, $projectDB, $dbForPlatform) - ->setPDO($register->get('db', true)) - ->execute(); - } catch (\Throwable $th) { - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - throw $th; - } - - $this->clearProjectsCache($project); - } - - $sum = \count($projects); - $projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($offset)]); - - $offset = $offset + $limit; - $count = $count + $sum; - - Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); - } - - Console::success('Data Migration Completed'); - } }