From ca6f9476427fb7a12ab95b6c95e7a222772689f7 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 28 Aug 2025 10:04:28 +0530 Subject: [PATCH 1/9] chore: update description for upsert methods --- app/config/specs/open-api3-1.8.x-console.json | 4 ++-- app/config/specs/open-api3-1.8.x-server.json | 4 ++-- app/config/specs/open-api3-latest-console.json | 4 ++-- app/config/specs/open-api3-latest-server.json | 4 ++-- app/config/specs/swagger2-1.8.x-console.json | 4 ++-- app/config/specs/swagger2-1.8.x-server.json | 4 ++-- app/config/specs/swagger2-latest-console.json | 4 ++-- app/config/specs/swagger2-latest-server.json | 4 ++-- .../Http/Databases/Collections/Documents/Bulk/Upsert.php | 2 +- .../Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php | 2 +- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/app/config/specs/open-api3-1.8.x-console.json b/app/config/specs/open-api3-1.8.x-console.json index 3812613df5..539463fecf 100644 --- a/app/config/specs/open-api3-1.8.x-console.json +++ b/app/config/specs/open-api3-1.8.x-console.json @@ -8940,7 +8940,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -36183,7 +36183,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-1.8.x-server.json b/app/config/specs/open-api3-1.8.x-server.json index 2028156c4c..d3a60a4ee4 100644 --- a/app/config/specs/open-api3-1.8.x-server.json +++ b/app/config/specs/open-api3-1.8.x-server.json @@ -8416,7 +8416,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -26542,7 +26542,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 3812613df5..539463fecf 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -8940,7 +8940,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -36183,7 +36183,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 2028156c4c..d3a60a4ee4 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -8416,7 +8416,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -26542,7 +26542,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/swagger2-1.8.x-console.json b/app/config/specs/swagger2-1.8.x-console.json index a6fe530d16..c0f9edde62 100644 --- a/app/config/specs/swagger2-1.8.x-console.json +++ b/app/config/specs/swagger2-1.8.x-console.json @@ -9058,7 +9058,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -36306,7 +36306,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-1.8.x-server.json b/app/config/specs/swagger2-1.8.x-server.json index 5b8c700365..4de1241c76 100644 --- a/app/config/specs/swagger2-1.8.x-server.json +++ b/app/config/specs/swagger2-1.8.x-server.json @@ -8524,7 +8524,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -26725,7 +26725,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index a6fe530d16..c0f9edde62 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -9058,7 +9058,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -36306,7 +36306,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 5b8c700365..4de1241c76 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -8524,7 +8524,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert Documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -26725,7 +26725,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert Rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index d7d6e1abbe..c72ff81735 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -39,7 +39,7 @@ class Upsert extends Action $this ->setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) ->setHttpPath('/v1/databases/:databaseId/collections/:collectionId/documents') - ->desc('Create or update documents') + ->desc('Upsert Documents') ->groups(['api', 'database']) ->label('scope', 'documents.write') ->label('resourceType', RESOURCE_TYPE_DATABASES) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php index 7b0d53da74..1876d8231d 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php @@ -30,7 +30,7 @@ class Upsert extends DocumentsUpsert $this ->setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) ->setHttpPath('/v1/tablesdb/:databaseId/tables/:tableId/rows') - ->desc('Create or update rows') + ->desc('Upsert Rows') ->groups(['api', 'database']) ->label('scope', ['rows.write', 'documents.write']) ->label('resourceType', RESOURCE_TYPE_DATABASES) From 6e519850375e1f92e9823bfa1deea33b60be3d1e Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 28 Aug 2025 12:31:14 +0530 Subject: [PATCH 2/9] casing --- app/config/specs/open-api3-1.8.x-console.json | 4 ++-- app/config/specs/open-api3-1.8.x-server.json | 4 ++-- app/config/specs/open-api3-latest-console.json | 4 ++-- app/config/specs/open-api3-latest-server.json | 4 ++-- app/config/specs/swagger2-1.8.x-console.json | 4 ++-- app/config/specs/swagger2-1.8.x-server.json | 4 ++-- app/config/specs/swagger2-latest-console.json | 4 ++-- app/config/specs/swagger2-latest-server.json | 4 ++-- .../Http/Databases/Collections/Documents/Bulk/Upsert.php | 2 +- .../Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php | 2 +- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/app/config/specs/open-api3-1.8.x-console.json b/app/config/specs/open-api3-1.8.x-console.json index 539463fecf..29c45692ec 100644 --- a/app/config/specs/open-api3-1.8.x-console.json +++ b/app/config/specs/open-api3-1.8.x-console.json @@ -8940,7 +8940,7 @@ } }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -36183,7 +36183,7 @@ } }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-1.8.x-server.json b/app/config/specs/open-api3-1.8.x-server.json index d3a60a4ee4..60c07da12a 100644 --- a/app/config/specs/open-api3-1.8.x-server.json +++ b/app/config/specs/open-api3-1.8.x-server.json @@ -8416,7 +8416,7 @@ } }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -26542,7 +26542,7 @@ } }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 539463fecf..29c45692ec 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -8940,7 +8940,7 @@ } }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -36183,7 +36183,7 @@ } }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index d3a60a4ee4..60c07da12a 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -8416,7 +8416,7 @@ } }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -26542,7 +26542,7 @@ } }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/swagger2-1.8.x-console.json b/app/config/specs/swagger2-1.8.x-console.json index c0f9edde62..5e121e8840 100644 --- a/app/config/specs/swagger2-1.8.x-console.json +++ b/app/config/specs/swagger2-1.8.x-console.json @@ -9058,7 +9058,7 @@ ] }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -36306,7 +36306,7 @@ ] }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-1.8.x-server.json b/app/config/specs/swagger2-1.8.x-server.json index 4de1241c76..e4ffbbe973 100644 --- a/app/config/specs/swagger2-1.8.x-server.json +++ b/app/config/specs/swagger2-1.8.x-server.json @@ -8524,7 +8524,7 @@ ] }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -26725,7 +26725,7 @@ ] }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index c0f9edde62..5e121e8840 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -9058,7 +9058,7 @@ ] }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -36306,7 +36306,7 @@ ] }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 4de1241c76..e4ffbbe973 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -8524,7 +8524,7 @@ ] }, "put": { - "summary": "Upsert Documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -26725,7 +26725,7 @@ ] }, "put": { - "summary": "Upsert Rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index c72ff81735..a6f27637e3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -39,7 +39,7 @@ class Upsert extends Action $this ->setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) ->setHttpPath('/v1/databases/:databaseId/collections/:collectionId/documents') - ->desc('Upsert Documents') + ->desc('Upsert documents') ->groups(['api', 'database']) ->label('scope', 'documents.write') ->label('resourceType', RESOURCE_TYPE_DATABASES) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php index 1876d8231d..c4a7c6e677 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php @@ -30,7 +30,7 @@ class Upsert extends DocumentsUpsert $this ->setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) ->setHttpPath('/v1/tablesdb/:databaseId/tables/:tableId/rows') - ->desc('Upsert Rows') + ->desc('Upsert rows') ->groups(['api', 'database']) ->label('scope', ['rows.write', 'documents.write']) ->label('resourceType', RESOURCE_TYPE_DATABASES) From 0737c78cb5a33156686131af5faca6ca9763e4b8 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 28 Aug 2025 19:06:18 +0530 Subject: [PATCH 3/9] added processing of migration resource stats --- src/Appwrite/Platform/Workers/Migrations.php | 103 ++++++++++++++++++- 1 file changed, 99 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 39adb37374..b90dcd635d 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; +use Appwrite\Event\StatsUsage; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -13,9 +14,14 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; +use Utopia\Database\Validator\Authorization as AuthorizationValidator; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; +use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Database as ResourceDatabase; +use Utopia\Migration\Resources\Database\Row as ResourceRow; +use Utopia\Migration\Resources\Database\Table as ResourceTable; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; use Utopia\Migration\Sources\CSV; @@ -45,6 +51,7 @@ class Migrations extends Action */ protected array $sourceReport = []; + private string $source; /** * @var callable */ @@ -69,13 +76,14 @@ class Migrations extends Action ->inject('logError') ->inject('queueForRealtime') ->inject('deviceForImports') + ->inject('queueForStatsUsage') // Added ->callback($this->action(...)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports, StatsUsage $queueForStatsUsage): void { $payload = $message->getPayload() ?? []; $this->deviceForImports = $deviceForImports; @@ -103,7 +111,7 @@ class Migrations extends Action return; } - $this->processMigration($migration, $queueForRealtime); + $this->processMigration($migration, $queueForRealtime, $queueForStatsUsage); // Updated } /** @@ -267,7 +275,7 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration, Realtime $queueForRealtime): void + protected function processMigration(Document $migration, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage): void { $project = $this->project; $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); @@ -301,6 +309,7 @@ class Migrations extends Action $destination ); + $aggregatedResources = []; /** Start Transfer */ if (empty($source->getErrors())) { $migration->setAttribute('stage', 'migrating'); @@ -308,9 +317,13 @@ class Migrations extends Action $transfer->run( $migration->getAttribute('resources'), - function () use ($migration, $transfer, $projectDocument, $queueForRealtime) { + function ($resources) use ($migration, $transfer, $projectDocument, $queueForRealtime, &$aggregatedResources) { $migration->setAttribute('resourceData', json_encode($transfer->getCache())); $migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); + + if (!empty($resources)) { + $aggregatedResources[] = $resources; + } $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); }, $migration->getAttribute('resourceId'), @@ -412,9 +425,91 @@ class Migrations extends Action } if ($migration->getAttribute('status', '') === 'completed') { + foreach ($aggregatedResources as $resource) { + $this->processMigrationResourceStats( + $resource, + $queueForStatsUsage, + $projectDocument, + $migration->getAttribute('source'), + $migration->getAttribute('resourceId') + ); + } $destination?->success(); $source?->success(); } } } + + private function processMigrationResourceStats(array $resources, StatsUsage $queueForStatsUsage, Document $projectDocument, string $source, ?string $resourceId) + { + /** + * @var Resource $resource + */ + var_dump($resources); + $resource = $resources[0]; + + $databaseInternalId = null; + $tableInternalId = null; + if ($source === CSV::getName() && !empty($resourceId)) { + [$databaseId, $tableId] = explode(':', $resourceId); + $database = AuthorizationValidator::skip(fn () => $this->dbForProject->getDocument('databases', $databaseId)); + $table = AuthorizationValidator::skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $tableId)); + $databaseInternalId = (int) $database->getSequence(); + $tableInternalId = (int) $table->getSequence(); + } + + $count = count($resources); + + switch ($resource->getName()) { + case ResourceDatabase::getName(): + $queueForStatsUsage->addMetric(METRIC_DATABASES, $count); + break; + + case ResourceTable::getName(): + /** @var ResourceTable $resource */ + $dbSeq = ($source === CSV::getName() && !empty($databaseInternalId)) + ? (int) $databaseInternalId + : (int) $resource->getDatabase()->getSequence(); + + $queueForStatsUsage + ->addMetric(METRIC_COLLECTIONS, $count) + ->addMetric( + str_replace('{databaseInternalId}', $dbSeq, METRIC_DATABASE_ID_COLLECTIONS), + $count + ); + break; + + case ResourceRow::getName(): + /** @var ResourceRow $resource */ + + $table = $resource->getTable(); + $dbSeq = ($source === CSV::getName() && !empty($databaseInternalId)) + ? (int) $databaseInternalId + : (int) $table->getDatabase()->getSequence(); + $colSeq = ($source === CSV::getName() && !empty($tableInternalId)) + ? (int) $tableInternalId + : (int) $table->getSequence(); + + $queueForStatsUsage + ->addMetric( + str_replace( + ['{databaseInternalId}','{collectionInternalId}'], + [$dbSeq, $colSeq], + METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS + ), + $count + ) + ->addMetric( + str_replace('{databaseInternalId}', $dbSeq, METRIC_DATABASE_ID_DOCUMENTS), + $count + ); + break; + + default: + break; + } + + $queueForStatsUsage->setProject($projectDocument)->trigger(); + $queueForStatsUsage->reset(); + } } From a809832f05bb992f7b5afc8887b4b9604c0e2f0b Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 28 Aug 2025 19:10:00 +0530 Subject: [PATCH 4/9] lint --- src/Appwrite/Platform/Workers/Migrations.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index b90dcd635d..2e23621850 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -76,7 +76,7 @@ class Migrations extends Action ->inject('logError') ->inject('queueForRealtime') ->inject('deviceForImports') - ->inject('queueForStatsUsage') // Added + ->inject('queueForStatsUsage') ->callback($this->action(...)); } @@ -111,7 +111,7 @@ class Migrations extends Action return; } - $this->processMigration($migration, $queueForRealtime, $queueForStatsUsage); // Updated + $this->processMigration($migration, $queueForRealtime, $queueForStatsUsage); } /** From 743bcceb5668b27ab3aba9e207b0de2a3a3181db Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 28 Aug 2025 19:13:24 +0530 Subject: [PATCH 5/9] updated migration version --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index 077f5204d1..1a776c089e 100644 --- a/composer.lock +++ b/composer.lock @@ -3557,16 +3557,16 @@ }, { "name": "utopia-php/database", - "version": "1.2.1", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "99beaf1dd6dc3561c8332f9893325777553644a4" + "reference": "8a536fead840d9da6ee819fe6b80e0f047997f69" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/99beaf1dd6dc3561c8332f9893325777553644a4", - "reference": "99beaf1dd6dc3561c8332f9893325777553644a4", + "url": "https://api.github.com/repos/utopia-php/database/zipball/8a536fead840d9da6ee819fe6b80e0f047997f69", + "reference": "8a536fead840d9da6ee819fe6b80e0f047997f69", "shasum": "" }, "require": { @@ -3607,9 +3607,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/1.2.1" + "source": "https://github.com/utopia-php/database/tree/1.2.3" }, - "time": "2025-08-26T16:05:26+00:00" + "time": "2025-08-27T11:47:04+00:00" }, { "name": "utopia-php/detector", @@ -4109,16 +4109,16 @@ }, { "name": "utopia-php/migration", - "version": "1.0.0", + "version": "1.0.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569" + "reference": "38171023efd3abe650d2abc5ac65f5df52311da6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/38171023efd3abe650d2abc5ac65f5df52311da6", + "reference": "38171023efd3abe650d2abc5ac65f5df52311da6", "shasum": "" }, "require": { @@ -4159,9 +4159,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.0.0" + "source": "https://github.com/utopia-php/migration/tree/1.0.1" }, - "time": "2025-08-13T09:15:53+00:00" + "time": "2025-08-28T13:41:25+00:00" }, { "name": "utopia-php/orchestration", From f067d1fadf3c805af3b03e8b35a0153899ef8b46 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 28 Aug 2025 19:14:14 +0530 Subject: [PATCH 6/9] updated reset of the stats usage worker to reset metrics --- src/Appwrite/Event/StatsUsage.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php index f6b1d695f4..f9e03c7c3d 100644 --- a/src/Appwrite/Event/StatsUsage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -85,4 +85,11 @@ class StatsUsage extends Event }), ]; } + + public function reset(): Event + { + $this->metrics = []; + parent::reset(); + return $this; + } } From e2575e73604fcc74ae08d4857f953793984541bb Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 28 Aug 2025 19:21:11 +0530 Subject: [PATCH 7/9] linting --- src/Appwrite/Platform/Workers/Migrations.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 2e23621850..f5def0fff6 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -445,7 +445,6 @@ class Migrations extends Action /** * @var Resource $resource */ - var_dump($resources); $resource = $resources[0]; $databaseInternalId = null; From bef0aff7c91dba50492ec762f4cebc106bfb51ea Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 28 Aug 2025 19:59:48 +0530 Subject: [PATCH 8/9] optimised the memory usage by resource aggregation --- src/Appwrite/Platform/Workers/Migrations.php | 66 +++++++++++--------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f5def0fff6..eb0ece0197 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -322,7 +322,34 @@ class Migrations extends Action $migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); if (!empty($resources)) { - $aggregatedResources[] = $resources; + /** + * @var Resource $resource + */ + $resource = $resources[0]; + $count = count($resources); + $databaseId = null; + $tableId = null; + switch ($resource->getName()) { + case ResourceTable::getName(): + /** @var ResourceTable $resource */ + $databaseId = $resource->getDatabase()->getSequence(); + break; + case ResourceRow::getName(): + /** @var ResourceRow $resource */ + $table = $resource->getTable(); + $databaseId = $table->getDatabase()->getSequence(); + $tableId = $table->getSequence(); + break; + default: + break; + } + $aggregatedResources[] = [ + "name" => $resource->getName(), + "count" => $count, + "databaseId" => $databaseId, + "tableId" => $tableId + ]; + } $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); }, @@ -442,14 +469,12 @@ class Migrations extends Action private function processMigrationResourceStats(array $resources, StatsUsage $queueForStatsUsage, Document $projectDocument, string $source, ?string $resourceId) { - /** - * @var Resource $resource - */ - $resource = $resources[0]; + $resourceName = $resources['name']; + $count = $resources['count']; + $databaseInternalId = $resources['databaseId']; + $tableInternalId = $resources['tableId']; - $databaseInternalId = null; - $tableInternalId = null; - if ($source === CSV::getName() && !empty($resourceId)) { + if ($source === CSV::getName()) { [$databaseId, $tableId] = explode(':', $resourceId); $database = AuthorizationValidator::skip(fn () => $this->dbForProject->getDocument('databases', $databaseId)); $table = AuthorizationValidator::skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $tableId)); @@ -457,49 +482,32 @@ class Migrations extends Action $tableInternalId = (int) $table->getSequence(); } - $count = count($resources); - - switch ($resource->getName()) { + switch ($resourceName) { case ResourceDatabase::getName(): $queueForStatsUsage->addMetric(METRIC_DATABASES, $count); break; case ResourceTable::getName(): - /** @var ResourceTable $resource */ - $dbSeq = ($source === CSV::getName() && !empty($databaseInternalId)) - ? (int) $databaseInternalId - : (int) $resource->getDatabase()->getSequence(); - $queueForStatsUsage ->addMetric(METRIC_COLLECTIONS, $count) ->addMetric( - str_replace('{databaseInternalId}', $dbSeq, METRIC_DATABASE_ID_COLLECTIONS), + str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $count ); break; case ResourceRow::getName(): - /** @var ResourceRow $resource */ - - $table = $resource->getTable(); - $dbSeq = ($source === CSV::getName() && !empty($databaseInternalId)) - ? (int) $databaseInternalId - : (int) $table->getDatabase()->getSequence(); - $colSeq = ($source === CSV::getName() && !empty($tableInternalId)) - ? (int) $tableInternalId - : (int) $table->getSequence(); - $queueForStatsUsage ->addMetric( str_replace( ['{databaseInternalId}','{collectionInternalId}'], - [$dbSeq, $colSeq], + [$databaseInternalId, $tableInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS ), $count ) ->addMetric( - str_replace('{databaseInternalId}', $dbSeq, METRIC_DATABASE_ID_DOCUMENTS), + str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $count ); break; From f43cbaf1a8d0e08e7b261706a63de4a251bacf9d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 29 Aug 2025 02:48:31 +1200 Subject: [PATCH 9/9] Update src/Appwrite/Platform/Workers/Migrations.php --- src/Appwrite/Platform/Workers/Migrations.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index eb0ece0197..ff25e799c1 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -344,10 +344,10 @@ class Migrations extends Action break; } $aggregatedResources[] = [ - "name" => $resource->getName(), - "count" => $count, - "databaseId" => $databaseId, - "tableId" => $tableId + 'name' => $resource->getName(), + 'count' => $count, + 'databaseId' => $databaseId, + 'tableId' => $tableId ]; }