Remove redundant doc migration

2026-05-23 17:08:45 +00:00 · 2025-05-16 17:25:06 +12:00 · 2025-05-16 17:25:06 +12:00 · 907d968a5f
commit 907d968a5f
parent 7344733165
2 changed files with 94 additions and 158 deletions
--- a/src/Appwrite/Migration/Version/V23.php
+++ b/src/Appwrite/Migration/Version/V23.php
@ -404,89 +404,38 @@ class V23 extends Migration
                17. Convert build's "path" to "buildPath"
                18. Convert build's "logs" to "buildLogs"
                19. Fill "totalSize" with "buildSize" plus "sourceSize"
-                --- Support for functions dual writing, if function's deploymentId is current deployment
-                1. Fill function's "deploymentCreatedAt" with deployment's "$createdAt"
-                --- Support for functions dual writing, if this is most recent deployment
-                2. Fill function's "latestDeploymentId" with deployment's "$id"
-                3. Fill function's "latestDeploymentInternalId" with deployment's "$internalId"
-                4. Fill function's "latestDeploymentCreatedAt" with deployment's "$createdAt"
-                5. Fill function's "latestDeploymentStatus" with deployment's "$status"
                */

                $document
-                    ->setAttribute("buildCommands", $document->getAttribute("commands"))
-                    ->setAttribute("sourcePath", $document->getAttribute("path"))
-                    ->setAttribute("sourceSize", $document->getAttribute("size"))
-                    ->setAttribute("sourceMetadata", $document->getAttribute("metadata"))
-                    ->setAttribute("sourceChunksTotal", $document->getAttribute("chunksTotal"))
-                    ->setAttribute("sourceChunksUploaded", $document->getAttribute("chunksUploaded"))
-                ;
+                    ->setAttribute('buildCommands', $document->getAttribute('commands'))
+                    ->setAttribute('sourcePath', $document->getAttribute('path'))
+                    ->setAttribute('sourceSize', $document->getAttribute('size'))
+                    ->setAttribute('sourceMetadata', $document->getAttribute('metadata'))
+                    ->setAttribute('sourceChunksTotal', $document->getAttribute('chunksTotal'))
+                    ->setAttribute('sourceChunksUploaded', $document->getAttribute('chunksUploaded'));

                $build = new Document();
-
                if (!empty($document->getAttribute('buildId'))) {
-                    $build = Authorization::skip(fn () => $this->projectDB->getDocument('builds', $document->getAttribute('buildId')));
+                    $build = $this->dbForProject->getDocument('builds', $document->getAttribute('buildId'));
                }

                $document
-                    ->setAttribute("buildStartedAt", $build->getAttribute("startTime", null))
-                    ->setAttribute("buildEndedAt", $build->getAttribute("endTime", null))
-                    ->setAttribute("buildDuration", $build->getAttribute("duration", 0))
-                    ->setAttribute("buildSize", $build->getAttribute("size", 0))
-                    ->setAttribute("status", $build->getAttribute("status", null))
-                    ->setAttribute("buildPath", $build->getAttribute("path", ""))
-                    ->setAttribute("buildLogs", $build->getAttribute("logs", ""))
-                ;
+                    ->setAttribute('buildStartedAt', $build->getAttribute('startTime'))
+                    ->setAttribute('buildEndedAt', $build->getAttribute('endTime'))
+                    ->setAttribute('buildDuration', $build->getAttribute('duration', 0))
+                    ->setAttribute('buildSize', $build->getAttribute('size', 0))
+                    ->setAttribute('status', $build->getAttribute('status'))
+                    ->setAttribute('buildPath', $build->getAttribute('path', ''))
+                    ->setAttribute('buildLogs', $build->getAttribute('logs', ''));

-                $totalSize = $document->getAttribute('buildSize', 0) + $document->getAttribute('sourceSize', 0);
-                $document->setAttribute("totalSize", $totalSize);
+                $totalSize = $document->getAttribute('buildSize', 0)
+                    + $document->getAttribute('sourceSize', 0);

-                $function = Authorization::skip(fn () => $this->projectDB->getDocument('functions', $document->getAttribute('resourceId')));
-                if (!$function->isEmpty()) {
-                    $activeDeploymentId = $function->getAttribute('deployment', $function->getAttribute('deploymentId', ''));
-                    if ($activeDeploymentId === $document->getId()) {
-                        $function->setAttribute('deploymentCreatedAt', $document->getCreatedAt());
-
-                        $function = Authorization::skip(fn () => $this->projectDB->updateDocument('functions', $function->getId(), $function));
-                    } else {
-                        $latestDeployments = Authorization::skip(fn () => $this->projectDB->find('deployments', [
-                            Query::orderDesc(),
-                            Query::limit(1),
-                            Query::equal('resourceType', ['functions']),
-                            Query::equal('resourceId', [$function->getId()]),
-                        ]));
-                        $latestDeployment = $latestDeployments[0] ? $latestDeployments[0] : new Document();
-
-                        if (!$latestDeployment->isEmpty()) {
-                            if ($latestDeployment->getId() === $document->getId()) {
-                                $function
-                                    ->setAttribute("latestDeploymentId", $document->getId())
-                                    ->setAttribute("latestDeploymentInternalId", $document->getInternalId())
-                                    ->setAttribute("latestDeploymentCreatedAt", $document->getCreatedAt())
-                                    ->setAttribute("latestDeploymentStatus", $document->getAttribute('status'))
-                                ;
-
-                                $function = Authorization::skip(fn () => $this->projectDB->updateDocument('functions', $function->getId(), $function));
-                            }
-                        }
-                    }
-                }
-                break;
-            case 'builds':
-                /**
-                    1. Trigger deployments document update
-                 */
-                if (!empty($document->getAttribute('deploymentId'))) {
-                    $deployment = Authorization::skip(fn () => $this->projectDB->getDocument('deployments', $document->getAttribute('deploymentId')));
-                    if (!$deployment->isEmpty()) {
-                        $deployment = $this->fixDocument($deployment);
-
-                        Authorization::skip(fn () => $this->projectDB->updateDocument('deployments', $deployment->getId(), $deployment));
-                    }
-                }
+                $document->setAttribute('totalSize', $totalSize);
                break;
+            case 'migrations':
                /*
-                    1. Fill "options" with "[]"
+                1. Fill "options" with "[]"
                */
                $document->setAttribute('options', []);
                break;
--- a/src/Appwrite/Platform/Tasks/Migrate.php
+++ b/src/Appwrite/Platform/Tasks/Migrate.php
@ -4,6 +4,7 @@ namespace Appwrite\Platform\Tasks;

 use Appwrite\Migration\Migration;
 use Redis;
+use Swoole\Runtime;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Database\Database;
@ -26,9 +27,10 @@ class Migrate extends Action

    public function __construct()
    {
+        Runtime::enableCoroutine();
+
        $this
            ->desc('Migrate Appwrite to new version')
-            /** @TODO APP_VERSION_STABLE needs to be defined */
            ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
            ->inject('dbForPlatform')
            ->inject('getProjectDB')
@ -36,12 +38,82 @@ class Migrate extends Action
            ->callback($this->action(...));
    }

-    private function clearProjectsCache(Document $project)
+    /**
+     * @param string $version
+     * @param Database $dbForPlatform
+     * @param callable(Document): Database $getProjectDB
+     * @param Registry $register
+     * @return void
+     */
+    public function action(
+        string $version,
+        Database $dbForPlatform,
+        callable $getProjectDB,
+        Registry $register,
+    ): void
+    {
+        Authorization::disable();
+
+        if (!\array_key_exists($version, Migration::$versions)) {
+            Console::error("Version {$version} not found.");
+            Console::exit(1);
+            return;
+        }
+
+        \Co\run(function (
+            string $version,
+            Database $dbForPlatform,
+            callable $getProjectDB,
+            Registry $register,
+        ) {
+            $this->redis = new Redis();
+            $this->redis->connect(
+                System::getEnv('_APP_REDIS_HOST', ''),
+                System::getEnv('_APP_REDIS_PORT', 6379),
+                3,
+                null,
+                10
+            );
+
+            Console::success('Starting Data Migration to version ' . $version);
+
+
+            $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
+
+            /** @var Migration $migration */
+            $migration = new $class();
+
+            $count = 0;
+            $total = $dbForPlatform->count('projects');
+
+            $dbForPlatform->foreach('projects', function (Document $project) use ($dbForPlatform, $getProjectDB, $register, $migration, &$count, $total) {
+                /** @var Database $dbForProject */
+                $dbForProject = $getProjectDB($project);
+                $dbForProject->disableValidation();
+
+                try {
+                    $migration
+                        ->setProject($project, $dbForProject, $dbForPlatform)
+                        ->setPDO($register->get('db', true))
+                        ->execute();
+                } catch (\Throwable $th) {
+                    Console::error('Failed to migrate project "' . $project->getId() . '" with error: ' . $th->getMessage());
+                    throw $th;
+                }
+
+                Console::log('Migrated ' . $count++ . '/' . $total . ' projects...');
+            });
+
+            Console::success('Migration completed');
+        }, [$version, $dbForPlatform, $getProjectDB, $register]);
+    }
+
+    private function clearProjectsCache(Document $project): void
    {
        try {
            $iterator = null;
            do {
-                $pattern = "default-cache-_{$project->getInternalId()}:*";
+                $pattern = "default-cache-mariadb:_{$project->getInternalId()}:*";
                $keys = $this->redis->scan($iterator, $pattern, 1000);
                if ($keys !== false) {
                    foreach ($keys as $key) {
@ -53,89 +125,4 @@ class Migrate extends Action
            Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage());
        }
    }
-
-    public function action(string $version, Database $dbForPlatform, callable $getProjectDB, Registry $register)
-    {
-        Authorization::disable();
-        if (!array_key_exists($version, Migration::$versions)) {
-            Console::error("Version {$version} not found.");
-            Console::exit(1);
-
-            return;
-        }
-
-        $this->redis = new Redis();
-        $this->redis->connect(
-            System::getEnv('_APP_REDIS_HOST', null),
-            System::getEnv('_APP_REDIS_PORT', 6379),
-            3,
-            null,
-            10
-        );
-
-        $app = new App('UTC');
-
-        Console::success('Starting Data Migration to version ' . $version);
-
-        $console = $app->getResource('console');
-
-        $limit = 30;
-        $sum = 30;
-        $offset = 0;
-        /**
-         * @var \Utopia\Database\Document[] $projects
-         */
-        $projects = [$console];
-        $count = 0;
-
-        try {
-            $totalProjects = $dbForPlatform->count('projects') + 1;
-        } catch (\Throwable $th) {
-            $dbForPlatform->setNamespace('_console');
-            $totalProjects = $dbForPlatform->count('projects') + 1;
-        }
-
-        $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
-        /** @var Migration $migration */
-        $migration = new $class();
-
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console' && $project->getInternalId() !== 'console') {
-                    continue;
-                }
-
-                $this->clearProjectsCache($project);
-
-                try {
-                    // TODO: Iterate through all project DBs
-                    /** @var Database $projectDB */
-                    $projectDB = $getProjectDB($project);
-                    $projectDB->disableValidation();
-                    $migration
-                        ->setProject($project, $projectDB, $dbForPlatform)
-                        ->setPDO($register->get('db', true))
-                        ->execute();
-                } catch (\Throwable $th) {
-                    Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
-                    throw $th;
-                }
-
-                $this->clearProjectsCache($project);
-            }
-
-            $sum = \count($projects);
-            $projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($offset)]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
-
-            Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...');
-        }
-
-        Console::success('Data Migration Completed');
-    }
 }