diff --git a/app/config/specs/open-api3-1.8.x-console.json b/app/config/specs/open-api3-1.8.x-console.json index 3812613df5..29c45692ec 100644 --- a/app/config/specs/open-api3-1.8.x-console.json +++ b/app/config/specs/open-api3-1.8.x-console.json @@ -8940,7 +8940,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -36183,7 +36183,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-1.8.x-server.json b/app/config/specs/open-api3-1.8.x-server.json index 2028156c4c..60c07da12a 100644 --- a/app/config/specs/open-api3-1.8.x-server.json +++ b/app/config/specs/open-api3-1.8.x-server.json @@ -8416,7 +8416,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -26542,7 +26542,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 3812613df5..29c45692ec 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -8940,7 +8940,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -36183,7 +36183,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 2028156c4c..60c07da12a 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -8416,7 +8416,7 @@ } }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "tags": [ "databases" @@ -26542,7 +26542,7 @@ } }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "tags": [ "tablesDB" diff --git a/app/config/specs/swagger2-1.8.x-console.json b/app/config/specs/swagger2-1.8.x-console.json index a6fe530d16..5e121e8840 100644 --- a/app/config/specs/swagger2-1.8.x-console.json +++ b/app/config/specs/swagger2-1.8.x-console.json @@ -9058,7 +9058,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -36306,7 +36306,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-1.8.x-server.json b/app/config/specs/swagger2-1.8.x-server.json index 5b8c700365..e4ffbbe973 100644 --- a/app/config/specs/swagger2-1.8.x-server.json +++ b/app/config/specs/swagger2-1.8.x-server.json @@ -8524,7 +8524,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -26725,7 +26725,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index a6fe530d16..5e121e8840 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -9058,7 +9058,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -36306,7 +36306,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 5b8c700365..e4ffbbe973 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -8524,7 +8524,7 @@ ] }, "put": { - "summary": "Create or update documents", + "summary": "Upsert documents", "operationId": "databasesUpsertDocuments", "consumes": [ "application\/json" @@ -26725,7 +26725,7 @@ ] }, "put": { - "summary": "Create or update rows", + "summary": "Upsert rows", "operationId": "tablesDBUpsertRows", "consumes": [ "application\/json" diff --git a/composer.lock b/composer.lock index 3bf17bc228..1a776c089e 100644 --- a/composer.lock +++ b/composer.lock @@ -4109,16 +4109,16 @@ }, { "name": "utopia-php/migration", - "version": "1.0.0", + "version": "1.0.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569" + "reference": "38171023efd3abe650d2abc5ac65f5df52311da6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/38171023efd3abe650d2abc5ac65f5df52311da6", + "reference": "38171023efd3abe650d2abc5ac65f5df52311da6", "shasum": "" }, "require": { @@ -4159,9 +4159,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.0.0" + "source": "https://github.com/utopia-php/migration/tree/1.0.1" }, - "time": "2025-08-13T09:15:53+00:00" + "time": "2025-08-28T13:41:25+00:00" }, { "name": "utopia-php/orchestration", diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php index f6b1d695f4..f9e03c7c3d 100644 --- a/src/Appwrite/Event/StatsUsage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -85,4 +85,11 @@ class StatsUsage extends Event }), ]; } + + public function reset(): Event + { + $this->metrics = []; + parent::reset(); + return $this; + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index d7d6e1abbe..a6f27637e3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -39,7 +39,7 @@ class Upsert extends Action $this ->setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) ->setHttpPath('/v1/databases/:databaseId/collections/:collectionId/documents') - ->desc('Create or update documents') + ->desc('Upsert documents') ->groups(['api', 'database']) ->label('scope', 'documents.write') ->label('resourceType', RESOURCE_TYPE_DATABASES) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php index 7b0d53da74..c4a7c6e677 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php @@ -30,7 +30,7 @@ class Upsert extends DocumentsUpsert $this ->setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) ->setHttpPath('/v1/tablesdb/:databaseId/tables/:tableId/rows') - ->desc('Create or update rows') + ->desc('Upsert rows') ->groups(['api', 'database']) ->label('scope', ['rows.write', 'documents.write']) ->label('resourceType', RESOURCE_TYPE_DATABASES) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 39adb37374..ff25e799c1 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; +use Appwrite\Event\StatsUsage; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -13,9 +14,14 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; +use Utopia\Database\Validator\Authorization as AuthorizationValidator; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; +use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Database as ResourceDatabase; +use Utopia\Migration\Resources\Database\Row as ResourceRow; +use Utopia\Migration\Resources\Database\Table as ResourceTable; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; use Utopia\Migration\Sources\CSV; @@ -45,6 +51,7 @@ class Migrations extends Action */ protected array $sourceReport = []; + private string $source; /** * @var callable */ @@ -69,13 +76,14 @@ class Migrations extends Action ->inject('logError') ->inject('queueForRealtime') ->inject('deviceForImports') + ->inject('queueForStatsUsage') ->callback($this->action(...)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports, StatsUsage $queueForStatsUsage): void { $payload = $message->getPayload() ?? []; $this->deviceForImports = $deviceForImports; @@ -103,7 +111,7 @@ class Migrations extends Action return; } - $this->processMigration($migration, $queueForRealtime); + $this->processMigration($migration, $queueForRealtime, $queueForStatsUsage); } /** @@ -267,7 +275,7 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration, Realtime $queueForRealtime): void + protected function processMigration(Document $migration, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage): void { $project = $this->project; $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); @@ -301,6 +309,7 @@ class Migrations extends Action $destination ); + $aggregatedResources = []; /** Start Transfer */ if (empty($source->getErrors())) { $migration->setAttribute('stage', 'migrating'); @@ -308,9 +317,40 @@ class Migrations extends Action $transfer->run( $migration->getAttribute('resources'), - function () use ($migration, $transfer, $projectDocument, $queueForRealtime) { + function ($resources) use ($migration, $transfer, $projectDocument, $queueForRealtime, &$aggregatedResources) { $migration->setAttribute('resourceData', json_encode($transfer->getCache())); $migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); + + if (!empty($resources)) { + /** + * @var Resource $resource + */ + $resource = $resources[0]; + $count = count($resources); + $databaseId = null; + $tableId = null; + switch ($resource->getName()) { + case ResourceTable::getName(): + /** @var ResourceTable $resource */ + $databaseId = $resource->getDatabase()->getSequence(); + break; + case ResourceRow::getName(): + /** @var ResourceRow $resource */ + $table = $resource->getTable(); + $databaseId = $table->getDatabase()->getSequence(); + $tableId = $table->getSequence(); + break; + default: + break; + } + $aggregatedResources[] = [ + 'name' => $resource->getName(), + 'count' => $count, + 'databaseId' => $databaseId, + 'tableId' => $tableId + ]; + + } $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); }, $migration->getAttribute('resourceId'), @@ -412,9 +452,71 @@ class Migrations extends Action } if ($migration->getAttribute('status', '') === 'completed') { + foreach ($aggregatedResources as $resource) { + $this->processMigrationResourceStats( + $resource, + $queueForStatsUsage, + $projectDocument, + $migration->getAttribute('source'), + $migration->getAttribute('resourceId') + ); + } $destination?->success(); $source?->success(); } } } + + private function processMigrationResourceStats(array $resources, StatsUsage $queueForStatsUsage, Document $projectDocument, string $source, ?string $resourceId) + { + $resourceName = $resources['name']; + $count = $resources['count']; + $databaseInternalId = $resources['databaseId']; + $tableInternalId = $resources['tableId']; + + if ($source === CSV::getName()) { + [$databaseId, $tableId] = explode(':', $resourceId); + $database = AuthorizationValidator::skip(fn () => $this->dbForProject->getDocument('databases', $databaseId)); + $table = AuthorizationValidator::skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $tableId)); + $databaseInternalId = (int) $database->getSequence(); + $tableInternalId = (int) $table->getSequence(); + } + + switch ($resourceName) { + case ResourceDatabase::getName(): + $queueForStatsUsage->addMetric(METRIC_DATABASES, $count); + break; + + case ResourceTable::getName(): + $queueForStatsUsage + ->addMetric(METRIC_COLLECTIONS, $count) + ->addMetric( + str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), + $count + ); + break; + + case ResourceRow::getName(): + $queueForStatsUsage + ->addMetric( + str_replace( + ['{databaseInternalId}','{collectionInternalId}'], + [$databaseInternalId, $tableInternalId], + METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS + ), + $count + ) + ->addMetric( + str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), + $count + ); + break; + + default: + break; + } + + $queueForStatsUsage->setProject($projectDocument)->trigger(); + $queueForStatsUsage->reset(); + } }