From 179def8f4d84921a6cd5efe58deeeb81c910470e Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 7 Apr 2025 10:12:58 +0530 Subject: [PATCH 01/20] add: imports collection --- app/config/collections/projects.php | 112 ++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 4461fcada6..ee2c6c0e71 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1881,4 +1881,116 @@ return [ ] ], ], + + 'imports' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('imports'), + 'name' => 'CSV Imports', + 'attributes' => [ + [ + '$id' => ID::custom('migrationId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('migrationInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('size'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('startedAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('resourceId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('resourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('error'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_status', + 'type' => Database::INDEX_KEY, + 'attributes' => ['status'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_resourceId', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ] ]; From 8dfbc128acc173e741c92f7954bc2fdf717aec27 Mon Sep 17 00:00:00 2001 From: Darshan Date: Tue, 8 Apr 2025 11:19:39 +0530 Subject: [PATCH 02/20] feat: import csv. --- Dockerfile | 2 +- app/config/collections/projects.php | 24 +++++- app/controllers/api/migrations.php | 90 ++++++++++++++++++++- docs/references/migrations/migration-csv.md | 1 + 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 docs/references/migrations/migration-csv.md diff --git a/Dockerfile b/Dockerfile index 88d5ed030b..1fdaaf2f0e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor COPY ./app /usr/src/code/app COPY ./public /usr/src/code/public COPY ./bin /usr/local/bin -COPY ./docs /usr/src/code/docs +#COPY ./docs /usr/src/code/docs COPY ./src /usr/src/code/src COPY ./dev /usr/src/code/dev diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index ee2c6c0e71..851b467159 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1848,7 +1848,29 @@ return [ 'default' => null, 'array' => false, 'filters' => [], - ] + ], + [ + '$id' => ID::custom('resourceId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('resourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 75afc7ed2c..10fc48bd33 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -1,5 +1,6 @@ dynamic($migration, Response::MODEL_MIGRATION); }); - App::post('/v1/migrations/firebase') ->groups(['api', 'migrations']) ->desc('Migrate Firebase data') @@ -290,6 +294,90 @@ App::post('/v1/migrations/nhost') ->dynamic($migration, Response::MODEL_MIGRATION); }); +App::post('/v1/migrations/csv') + ->groups(['api', 'migrations']) + ->desc('Import documents from a CSV') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + name: 'createCsvMigration', + description: '/docs/references/migrations/migration-csv.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->param('resourceId', null, new UID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->inject('$queueForEvents') + ->inject('queueForMigrations') + ->action(function (string $bucketId, string $fileId, string $resourceId, Request $request, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Migration $queueForEvents, Migration $queueForMigrations) { + + // TODO: Check if there's already a migrations worker process running for CSV Import for the same collection. + // If so, short-circuit and cancel the task early on because console may not allow it but API will. + // if (inProgress(resourceId)) { + // throw some exception. + //} + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + // TODO: send path migrations/csv worker + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + // TODO: add stuff to migration library + 'source' => SourcesCSV::getName(), + 'destination' => DestinationsCSV::getName(), + 'resources' => [Resource::TYPE_DOCUMENT], + 'resourceId' => $resourceId, + 'resourceType' => Resource::TYPE_DATABASE, + 'statusCounters' => [], + 'resourceData' => [], + 'errors' => [], + 'credentials' => [], + ])); + + // TODO: use migrationId or importId? + $queueForEvents->setParam('migrationId', $migration->getId()); + + // Trigger Import + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->trigger(); + }); + App::get('/v1/migrations') ->groups(['api', 'migrations']) ->desc('List migrations') diff --git a/docs/references/migrations/migration-csv.md b/docs/references/migrations/migration-csv.md new file mode 100644 index 0000000000..7a32d5ff6e --- /dev/null +++ b/docs/references/migrations/migration-csv.md @@ -0,0 +1 @@ +Import documents from a CSV file into your Appwrite database. This endpoint allows you to import documents from a CSV file uploaded to Appwrite Storage bucket. \ No newline at end of file From 89815cba1c11193b2bc9e9ea77168df53541e24c Mon Sep 17 00:00:00 2001 From: Darshan Date: Tue, 8 Apr 2025 11:20:47 +0530 Subject: [PATCH 03/20] revert: local dockerfile change. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1fdaaf2f0e..88d5ed030b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor COPY ./app /usr/src/code/app COPY ./public /usr/src/code/public COPY ./bin /usr/local/bin -#COPY ./docs /usr/src/code/docs +COPY ./docs /usr/src/code/docs COPY ./src /usr/src/code/src COPY ./dev /usr/src/code/dev From 624538fcaf5d0c99f86f71e8cede48b3e95d402a Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 9 Apr 2025 16:35:27 +0530 Subject: [PATCH 04/20] revert: local dockerfile change. --- Dockerfile | 2 + app/config/collections/projects.php | 17 +++- app/controllers/api/migrations.php | 102 ++++++++++++++----- app/init/constants.php | 1 + app/init/resources.php | 4 + app/worker.php | 4 + composer.json | 8 +- composer.lock | 54 ++++++---- docker-compose.yml | 6 +- src/Appwrite/Platform/Workers/Migrations.php | 40 +++++++- 10 files changed, 182 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 88d5ed030b..4b5ac3fc62 100755 --- a/Dockerfile +++ b/Dockerfile @@ -44,12 +44,14 @@ COPY ./dev /usr/src/code/dev # Set Volumes RUN mkdir -p /storage/uploads && \ + mkdir -p /storage/csv-imports && \ mkdir -p /storage/cache && \ mkdir -p /storage/config && \ mkdir -p /storage/certificates && \ mkdir -p /storage/functions && \ mkdir -p /storage/debug && \ chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ + chown -Rf www-data.www-data /storage/csv-imports && chmod -Rf 0755 /storage/csv-imports && \ chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 851b467159..d28b2a7e38 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1894,6 +1894,13 @@ return [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_resource_id', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_DESC], + ], [ '$id' => ID::custom('_fulltext_search'), 'type' => Database::INDEX_FULLTEXT, @@ -1935,7 +1942,7 @@ return [ '$id' => ID::custom('status'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 16, + 'size' => Database::LENGTH_KEY, 'signed' => true, 'required' => false, 'default' => null, @@ -1987,14 +1994,14 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('error'), + '$id' => ID::custom('errors'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 65535, 'signed' => true, - 'required' => false, + 'required' => true, 'default' => null, - 'array' => false, + 'array' => true, 'filters' => [], ], ], diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 10fc48bd33..90f0930ff2 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -4,12 +4,12 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Event; use Appwrite\Event\Migration; use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Validator\Queries\Migrations; -use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\App; use Utopia\Database\Database; @@ -22,10 +22,16 @@ use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; +use Utopia\Migration\Sources\Csv; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; +use Utopia\Migration\Transfer; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; +use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; +use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Integer; use Utopia\Validator\Text; @@ -314,28 +320,38 @@ App::post('/v1/migrations/csv') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') - ->param('resourceId', null, new UID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') - ->inject('request') + ->param('resourceId', null, new Text(75), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') ->inject('response') ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('deviceForLocal') - ->inject('$queueForEvents') + ->inject('deviceForCsvImports') + ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, Request $request, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Migration $queueForEvents, Migration $queueForMigrations) { - - // TODO: Check if there's already a migrations worker process running for CSV Import for the same collection. - // If so, short-circuit and cancel the task early on because console may not allow it but API will. - // if (inProgress(resourceId)) { - // throw some exception. - //} - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForCsvImports, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + // Check if migration/import is already in progress! + // if for some reason the worker crashes, the stage will always be `init`, what do we do? + $isInProgress = Authorization::skip(function () use ($dbForProject, $resourceId) { + $exists = $dbForProject->findOne( + 'migrations', + [ + Query::notEqual('stage', 'finished'), + Query::equal('resourceId', [$resourceId]), + ] + ); + + return !$exists->isEmpty(); + }); + + if ($isInProgress || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::MIGRATION_IN_PROGRESS, 'An import is already in progress for this collection.'); + } + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } @@ -346,36 +362,72 @@ App::post('/v1/migrations/csv') } $path = $file->getAttribute('path', ''); - if (!$deviceForFiles->exists($path)) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - // TODO: send path migrations/csv worker + // read file content. + $source = $deviceForFiles->read($path); + + // decrypt + if (!empty($file->getAttribute('openSSLCipher'))) { + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + // decompress + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + // copy to temporary folder + $migrationId = ID::unique(); + $path = $deviceForCsvImports->getRoot() . '/' . $migrationId . '_' . $fileId . '.csv'; + $deviceForCsvImports->write($path, $source, 'text/csv'); + $fileSize = $deviceForCsvImports->getFileSize($path); + $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + $migration = $dbForProject->createDocument('migrations', new Document([ - '$id' => ID::unique(), + '$id' => $migrationId, 'status' => 'pending', 'stage' => 'init', - // TODO: add stuff to migration library - 'source' => SourcesCSV::getName(), - 'destination' => DestinationsCSV::getName(), - 'resources' => [Resource::TYPE_DOCUMENT], + 'source' => Csv::getName(), + 'destination' => Appwrite::class::getName(), + 'resources' => $resources, 'resourceId' => $resourceId, 'resourceType' => Resource::TYPE_DATABASE, 'statusCounters' => [], 'resourceData' => [], 'errors' => [], - 'credentials' => [], + 'credentials' => [ + 'path' => $path, + 'size' => $fileSize, + ], ])); - // TODO: use migrationId or importId? $queueForEvents->setParam('migrationId', $migration->getId()); - // Trigger Import $queueForMigrations ->setMigration($migration) ->setProject($project) ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); }); App::get('/v1/migrations') diff --git a/app/init/constants.php b/app/init/constants.php index 5e4edfd97d..2deeff1c95 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -49,6 +49,7 @@ const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; +const APP_STORAGE_CSV_IMPORTS = '/storage/csv-imports'; // Temporary storage for csv imports const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` diff --git a/app/init/resources.php b/app/init/resources.php index 2360179913..6eae238fee 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -508,6 +508,10 @@ App::setResource('deviceForFiles', function ($project) { return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); +App::setResource('deviceForCsvImports', function (Document $project) { + return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); +}, ['project']); + App::setResource('deviceForFunctions', function ($project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/worker.php b/app/worker.php index 6a51ee55be..4e865858a0 100644 --- a/app/worker.php +++ b/app/worker.php @@ -339,6 +339,10 @@ Server::setResource('pools', function (Registry $register) { return $register->get('pools'); }, ['register']); +Server::setResource('deviceForCsvImports', function (Document $project) { + return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); +}, ['project']); + Server::setResource('deviceForFunctions', function (Document $project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/composer.json b/composer.json index 3920351c06..4c26b19d1e 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.8.*", + "utopia-php/migration": "dev-feat-csv", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", @@ -91,6 +91,12 @@ "laravel/pint": "1.*", "phpbench/phpbench": "1.*" }, + "repositories": [ + { + "type": "git", + "url": "https://github.com/utopia-php/migration" + } + ], "provide": { "ext-phpiredis": "*" }, diff --git a/composer.lock b/composer.lock index 8db4706bb5..63323ad236 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6a54c8bc4f9f14cd3883f55880864630", + "content-hash": "cfb5c437126bf194a6fe7225961c1582", "packages": [ { "name": "adhocore/jwt", @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0" + "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/37eec0fe47ddd627911f318f29b6cd48196be0c0", - "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/0e7804c176c4b09d95b7985400aa38ce544cb7fc", + "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc", "shasum": "" }, "require": { @@ -1451,7 +1451,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-29T21:40:28+00:00" + "time": "2025-04-08T09:55:41+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3951,17 +3951,11 @@ }, { "name": "utopia-php/migration", - "version": "0.8.4", + "version": "dev-feat-csv", "source": { "type": "git", - "url": "https://github.com/utopia-php/migration.git", - "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/845fd04ccf5e0edb03c184b864e0596080a432b8", - "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8", - "shasum": "" + "url": "https://github.com/utopia-php/migration", + "reference": "5c4e6c61c393d176348e88b65730a14b52f4ea2e" }, "require": { "appwrite/appwrite": "11.*", @@ -3987,7 +3981,25 @@ "Utopia\\Migration\\": "src/Migration" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Migration" + } + }, + "scripts": { + "test": [ + "./vendor/bin/phpunit" + ], + "lint": [ + "./vendor/bin/pint --test" + ], + "format": [ + "./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" + ] + }, "license": [ "MIT" ], @@ -3999,11 +4011,7 @@ "upf", "utopia" ], - "support": { - "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.4" - }, - "time": "2025-03-28T02:08:22+00:00" + "time": "2025-04-08T08:38:41+00:00" }, { "name": "utopia-php/orchestration", @@ -8126,7 +8134,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/migration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 8c8a364f30..4181cc6564 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw + - appwrite-csv-imports:/storage/csv-imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -204,7 +205,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.2.53 + image: appwrite/console:5.2.56 restart: unless-stopped networks: - appwrite @@ -672,6 +673,8 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests + # for csv import access + - appwrite-csv-imports:/storage/csv-imports:rw depends_on: - mariadb environment: @@ -1132,6 +1135,7 @@ volumes: appwrite-redis: appwrite-cache: appwrite-uploads: + appwrite-csv-imports: appwrite-certificates: appwrite-functions: appwrite-builds: diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 4939dc8143..b670c79822 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,10 +4,12 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; +use Appwrite\ID; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; @@ -18,12 +20,14 @@ use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; +use Utopia\Migration\Sources\Csv; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Device; use Utopia\System\System; class Migrations extends Action @@ -32,6 +36,8 @@ class Migrations extends Action protected Database $dbForPlatform; + protected Device $deviceForCsvImports; + protected Document $project; /** @@ -57,15 +63,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') - ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime)); + ->inject('deviceForCsvImports') + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime, $deviceForCsvImports)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports): void { $payload = $message->getPayload() ?? []; + $this->deviceForCsvImports = $deviceForCsvImports; if (empty($payload)) { throw new Exception('Missing payload'); @@ -99,6 +107,7 @@ class Migrations extends Action protected function processSource(Document $migration): Source { $source = $migration->getAttribute('source'); + $resourceId = $migration->getAttribute('resourceId'); $credentials = $migration->getAttribute('credentials'); return match ($source) { @@ -128,6 +137,11 @@ class Migrations extends Action $credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey'], ), + Csv::getName() => new Csv( + $resourceId, + $credentials['path'], + $this->deviceForCsvImports + ), default => throw new \Exception('Invalid source type'), }; } @@ -222,8 +236,23 @@ class Migrations extends Action $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); $tempAPIKey = $this->generateAPIKey($projectDocument); + $importDocument = null; $transfer = $source = $destination = null; + if ($migration->getAttribute('source') === Csv::getName()) { + $fileSize = $migration->getAttribute('credentials', [])['size'] ?? 0; + $importDocument = new Document([ + '$id' => ID::unique(), + 'size' => $fileSize, // uncompressed and decrypted file size + 'startedAt' => DateTime::now(), + 'migrationId' => $migration->getId(), + 'migrationInternalId' => $migration->getInternalId(), + 'resourceId' => $migration->getAttribute('resourceId', ''), + 'resourceType' => $migration->getAttribute('resourceType', ''), + 'errors' => [], + ]); + } + try { if ( $migration->getAttribute('source') === SourceAppwrite::getName() && @@ -337,6 +366,7 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); + $importDocument?->setAttribute('errors', $errorMessages); } } finally { $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); @@ -379,6 +409,12 @@ class Migrations extends Action $destination?->success(); $source?->success(); } + + if ($migration->getAttribute('source') === Csv::getName()) { + // make and save the import document to database + $importDocument->setAttribute('status', $migration->getAttribute('status', '')); + $this->dbForProject->createDocument('imports', $importDocument); + } } } } From a53eb6c7500c305588f7dbb504e9b6698355edd9 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 9 Apr 2025 18:24:59 +0530 Subject: [PATCH 05/20] update: bump migrations and update params. --- composer.lock | 4 ++-- src/Appwrite/Platform/Workers/Migrations.php | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/composer.lock b/composer.lock index 63323ad236..9d9f0a78c7 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "5c4e6c61c393d176348e88b65730a14b52f4ea2e" + "reference": "b308d9183f1f8ab32e172f31744ad93db319cba9" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-08T08:38:41+00:00" + "time": "2025-04-09T12:54:03+00:00" }, { "name": "utopia-php/orchestration", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index b670c79822..ac13d54959 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -140,7 +140,8 @@ class Migrations extends Action Csv::getName() => new Csv( $resourceId, $credentials['path'], - $this->deviceForCsvImports + $this->deviceForCsvImports, + $this->dbForProject ), default => throw new \Exception('Invalid source type'), }; From a64b8e57af222e7178ffbd8b50e7b5715912b8af Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 12 Apr 2025 17:10:06 +0530 Subject: [PATCH 06/20] address comments. --- app/config/collections/projects.php | 4 ++-- app/controllers/api/migrations.php | 21 ++------------------- composer.lock | 4 ++-- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index d28b2a7e38..1fa474ff44 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1953,8 +1953,8 @@ return [ '$id' => ID::custom('size'), 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => 0, - 'signed' => true, + 'size' => 8, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 90f0930ff2..fbd9b3eb13 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -9,6 +9,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Validator\CompoundUID; use Appwrite\Utopia\Database\Validator\Queries\Migrations; use Appwrite\Utopia\Response; use Utopia\App; @@ -320,7 +321,7 @@ App::post('/v1/migrations/csv') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') - ->param('resourceId', null, new Text(75), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') ->inject('response') ->inject('dbForProject') ->inject('project') @@ -332,24 +333,6 @@ App::post('/v1/migrations/csv') $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - // Check if migration/import is already in progress! - // if for some reason the worker crashes, the stage will always be `init`, what do we do? - $isInProgress = Authorization::skip(function () use ($dbForProject, $resourceId) { - $exists = $dbForProject->findOne( - 'migrations', - [ - Query::notEqual('stage', 'finished'), - Query::equal('resourceId', [$resourceId]), - ] - ); - - return !$exists->isEmpty(); - }); - - if ($isInProgress || (!$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::MIGRATION_IN_PROGRESS, 'An import is already in progress for this collection.'); - } - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { diff --git a/composer.lock b/composer.lock index 9d9f0a78c7..270ceb6f8b 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "b308d9183f1f8ab32e172f31744ad93db319cba9" + "reference": "9847a1387136574c7539872232765c5c4d8064f7" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-09T12:54:03+00:00" + "time": "2025-04-12T11:12:09+00:00" }, { "name": "utopia-php/orchestration", From 2ad967043c7b17fd2cbba0e1e94585dba0761220 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 12 Apr 2025 17:10:29 +0530 Subject: [PATCH 07/20] update: deps. --- composer.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.lock b/composer.lock index 270ceb6f8b..5369b6da5a 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "9847a1387136574c7539872232765c5c4d8064f7" + "reference": "52fb030dfb988233dee96845b9c481542fe1a360" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-12T11:12:09+00:00" + "time": "2025-04-12T11:28:18+00:00" }, { "name": "utopia-php/orchestration", From 91199e746bba1fa24e7558a4fa8c5f26759355f3 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 14 Apr 2025 15:03:30 +0530 Subject: [PATCH 08/20] update: use plain csv, use `options`. --- app/config/collections/projects.php | 123 ++----------------- app/controllers/api/migrations.php | 42 ++----- src/Appwrite/Platform/Workers/Migrations.php | 27 +--- 3 files changed, 22 insertions(+), 170 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 1fa474ff44..688153dc29 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1794,6 +1794,17 @@ return [ 'array' => false, 'filters' => ['json', 'encrypt'], ], + [ + '$id' => ID::custom('options'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65536, + 'signed' => true, + 'required' => false, + 'default' => [], + 'array' => false, + 'filters' => ['json'], + ], [ '$id' => ID::custom('resources'), 'type' => Database::VAR_STRING, @@ -1910,116 +1921,4 @@ return [ ] ], ], - - 'imports' => [ - '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('imports'), - 'name' => 'CSV Imports', - 'attributes' => [ - [ - '$id' => ID::custom('migrationId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('migrationInternalId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('status'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('size'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 8, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('startedAt'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('resourceId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('resourceType'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('errors'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 65535, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => true, - 'filters' => [], - ], - ], - 'indexes' => [ - [ - '$id' => '_key_status', - 'type' => Database::INDEX_KEY, - 'attributes' => ['status'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_resourceId', - 'type' => Database::INDEX_KEY, - 'attributes' => ['resourceId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - ], - ] ]; diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index fbd9b3eb13..01c7a260f2 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -4,7 +4,6 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Event; use Appwrite\Event\Migration; use Appwrite\Extend\Exception; -use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; @@ -28,11 +27,8 @@ use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; -use Utopia\Storage\Compression\Algorithms\GZIP; -use Utopia\Storage\Compression\Algorithms\Zstd; use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; -use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Integer; use Utopia\Validator\Text; @@ -349,37 +345,17 @@ App::post('/v1/migrations/csv') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - // read file content. - $source = $deviceForFiles->read($path); - - // decrypt - if (!empty($file->getAttribute('openSSLCipher'))) { - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - // decompress - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; + if (!empty($file->getAttribute('openSSLCipher')) || $file->getAttribute('algorithm', Compression::NONE) !== Compression::NONE) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only uncompressed, unencrypted CSV files can be used for document import."); } // copy to temporary folder $migrationId = ID::unique(); - $path = $deviceForCsvImports->getRoot() . '/' . $migrationId . '_' . $fileId . '.csv'; - $deviceForCsvImports->write($path, $source, 'text/csv'); + $newPath = $deviceForCsvImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); + if (!$deviceForFiles->transfer($path, $newPath, $deviceForCsvImports)) { + throw new \Exception("Unable to copy file"); + } + $fileSize = $deviceForCsvImports->getFileSize($path); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); @@ -395,8 +371,8 @@ App::post('/v1/migrations/csv') 'statusCounters' => [], 'resourceData' => [], 'errors' => [], - 'credentials' => [ - 'path' => $path, + 'options' => [ + 'path' => $newPath, 'size' => $fileSize, ], ])); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index ac13d54959..417a42fa9c 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,12 +4,10 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; -use Appwrite\ID; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; @@ -109,6 +107,7 @@ class Migrations extends Action $source = $migration->getAttribute('source'); $resourceId = $migration->getAttribute('resourceId'); $credentials = $migration->getAttribute('credentials'); + $migrationOptions = $migration->getAttribute('options'); return match ($source) { Firebase::getName() => new Firebase( @@ -139,7 +138,7 @@ class Migrations extends Action ), Csv::getName() => new Csv( $resourceId, - $credentials['path'], + $migrationOptions['path'], $this->deviceForCsvImports, $this->dbForProject ), @@ -237,23 +236,8 @@ class Migrations extends Action $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); $tempAPIKey = $this->generateAPIKey($projectDocument); - $importDocument = null; $transfer = $source = $destination = null; - if ($migration->getAttribute('source') === Csv::getName()) { - $fileSize = $migration->getAttribute('credentials', [])['size'] ?? 0; - $importDocument = new Document([ - '$id' => ID::unique(), - 'size' => $fileSize, // uncompressed and decrypted file size - 'startedAt' => DateTime::now(), - 'migrationId' => $migration->getId(), - 'migrationInternalId' => $migration->getInternalId(), - 'resourceId' => $migration->getAttribute('resourceId', ''), - 'resourceType' => $migration->getAttribute('resourceType', ''), - 'errors' => [], - ]); - } - try { if ( $migration->getAttribute('source') === SourceAppwrite::getName() && @@ -367,7 +351,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $importDocument?->setAttribute('errors', $errorMessages); } } finally { $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); @@ -410,12 +393,6 @@ class Migrations extends Action $destination?->success(); $source?->success(); } - - if ($migration->getAttribute('source') === Csv::getName()) { - // make and save the import document to database - $importDocument->setAttribute('status', $migration->getAttribute('status', '')); - $this->dbForProject->createDocument('imports', $importDocument); - } } } } From ab73a70c83584cacec739e446a004aa3f4e94b6d Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 14 Apr 2025 16:30:05 +0530 Subject: [PATCH 09/20] address comments. --- Dockerfile | 4 ++-- app/init/constants.php | 2 +- docker-compose.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b5ac3fc62..3f85078152 100755 --- a/Dockerfile +++ b/Dockerfile @@ -44,14 +44,14 @@ COPY ./dev /usr/src/code/dev # Set Volumes RUN mkdir -p /storage/uploads && \ - mkdir -p /storage/csv-imports && \ + mkdir -p /storage/imports && \ mkdir -p /storage/cache && \ mkdir -p /storage/config && \ mkdir -p /storage/certificates && \ mkdir -p /storage/functions && \ mkdir -p /storage/debug && \ chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ - chown -Rf www-data.www-data /storage/csv-imports && chmod -Rf 0755 /storage/csv-imports && \ + chown -Rf www-data.www-data /storage/imports && chmod -Rf 0755 /storage/imports && \ chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ diff --git a/app/init/constants.php b/app/init/constants.php index 2deeff1c95..cdfb0b4d03 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -49,7 +49,7 @@ const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; -const APP_STORAGE_CSV_IMPORTS = '/storage/csv-imports'; // Temporary storage for csv imports +const APP_STORAGE_CSV_IMPORTS = '/storage/imports'; // Temporary storage for csv imports const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` diff --git a/docker-compose.yml b/docker-compose.yml index 4181cc6564..13a4136ca6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw - - appwrite-csv-imports:/storage/csv-imports:rw + - appwrite-csv-imports:/storage/imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -674,7 +674,7 @@ services: - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests # for csv import access - - appwrite-csv-imports:/storage/csv-imports:rw + - appwrite-csv-imports:/storage/imports:rw depends_on: - mariadb environment: From 73b2a14ea31f4d503d8f045dc1f435907dc510b5 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 11:55:04 +0530 Subject: [PATCH 10/20] update: changes as per migrations lib. --- app/controllers/api/migrations.php | 4 ++-- composer.lock | 4 ++-- src/Appwrite/Platform/Workers/Migrations.php | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 01c7a260f2..7dd8fe42f5 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -22,7 +22,7 @@ use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; -use Utopia\Migration\Sources\Csv; +use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; @@ -363,7 +363,7 @@ App::post('/v1/migrations/csv') '$id' => $migrationId, 'status' => 'pending', 'stage' => 'init', - 'source' => Csv::getName(), + 'source' => CSV::getName(), 'destination' => Appwrite::class::getName(), 'resources' => $resources, 'resourceId' => $resourceId, diff --git a/composer.lock b/composer.lock index 5369b6da5a..c25ec33591 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "52fb030dfb988233dee96845b9c481542fe1a360" + "reference": "9088ef1079da3fb13b8abc3821feee618475a0c3" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-12T11:28:18+00:00" + "time": "2025-04-16T06:21:15+00:00" }, { "name": "utopia-php/orchestration", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 417a42fa9c..c4182f8deb 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -18,7 +18,7 @@ use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; -use Utopia\Migration\Sources\Csv; +use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; @@ -136,7 +136,7 @@ class Migrations extends Action $credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey'], ), - Csv::getName() => new Csv( + CSV::getName() => new CSV( $resourceId, $migrationOptions['path'], $this->deviceForCsvImports, From 9d6cd8249ac5e3b542ab3cfd402c351e70f57441 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 11:56:40 +0530 Subject: [PATCH 11/20] change: constant name. --- app/init/constants.php | 2 +- app/init/resources.php | 2 +- app/worker.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index cdfb0b4d03..d22d4b3667 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -49,7 +49,7 @@ const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; -const APP_STORAGE_CSV_IMPORTS = '/storage/imports'; // Temporary storage for csv imports +const APP_STORAGE_IMPORTS = '/storage/imports'; // Temporary storage for csv imports const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` diff --git a/app/init/resources.php b/app/init/resources.php index 6eae238fee..6aceaf6fb4 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -509,7 +509,7 @@ App::setResource('deviceForFiles', function ($project) { }, ['project']); App::setResource('deviceForCsvImports', function (Document $project) { - return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); + return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); App::setResource('deviceForFunctions', function ($project) { diff --git a/app/worker.php b/app/worker.php index 4e865858a0..cd3b433d44 100644 --- a/app/worker.php +++ b/app/worker.php @@ -340,7 +340,7 @@ Server::setResource('pools', function (Registry $register) { }, ['register']); Server::setResource('deviceForCsvImports', function (Document $project) { - return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); + return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); Server::setResource('deviceForFunctions', function (Document $project) { From bc8683ab757cf0dc311c809d90b115f1a313628b Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 16:52:23 +0530 Subject: [PATCH 12/20] add: csv import tests! --- composer.json | 8 +- composer.lock | 44 ++-- .../Services/Migrations/MigrationsBase.php | 226 ++++++++++++++++++ tests/resources/documents.csv | 101 ++++++++ 4 files changed, 345 insertions(+), 34 deletions(-) create mode 100644 tests/resources/documents.csv diff --git a/composer.json b/composer.json index 4c26b19d1e..d0f3803210 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "dev-feat-csv", + "utopia-php/migration": "0.9.0", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", @@ -91,12 +91,6 @@ "laravel/pint": "1.*", "phpbench/phpbench": "1.*" }, - "repositories": [ - { - "type": "git", - "url": "https://github.com/utopia-php/migration" - } - ], "provide": { "ext-phpiredis": "*" }, diff --git a/composer.lock b/composer.lock index c25ec33591..46353a4a11 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cfb5c437126bf194a6fe7225961c1582", + "content-hash": "85afadfc660334537aaba2c355f98b9c", "packages": [ { "name": "adhocore/jwt", @@ -3951,11 +3951,17 @@ }, { "name": "utopia-php/migration", - "version": "dev-feat-csv", + "version": "0.9.0", "source": { "type": "git", - "url": "https://github.com/utopia-php/migration", - "reference": "9088ef1079da3fb13b8abc3821feee618475a0c3" + "url": "https://github.com/utopia-php/migration.git", + "reference": "545705e251b766940d2833893f267975d73abe32" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/545705e251b766940d2833893f267975d73abe32", + "reference": "545705e251b766940d2833893f267975d73abe32", + "shasum": "" }, "require": { "appwrite/appwrite": "11.*", @@ -3981,25 +3987,7 @@ "Utopia\\Migration\\": "src/Migration" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Migration" - } - }, - "scripts": { - "test": [ - "./vendor/bin/phpunit" - ], - "lint": [ - "./vendor/bin/pint --test" - ], - "format": [ - "./vendor/bin/pint" - ], - "check": [ - "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -4011,7 +3999,11 @@ "upf", "utopia" ], - "time": "2025-04-16T06:21:15+00:00" + "support": { + "issues": "https://github.com/utopia-php/migration/issues", + "source": "https://github.com/utopia-php/migration/tree/0.9.0" + }, + "time": "2025-04-16T07:52:53+00:00" }, { "name": "utopia-php/orchestration", @@ -8134,9 +8126,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/migration": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 6c468ee730..f7e7d20e99 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -10,8 +10,10 @@ use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Query; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; +use Utopia\Migration\Sources\CSV; trait MigrationsBase { @@ -896,4 +898,228 @@ trait MigrationsBase 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } + + /** + * Import documents from a CSV file. + */ + public function testCreateCsvMigration(): array + { + // make a database + $response = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('Test Database', $response['body']['name']); + + $databaseId = $response['body']['$id']; + + // make a collection + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'Test collection', + 'collectionId' => ID::unique(), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals($response['body']['name'], 'Test collection'); + + $collectionId = $response['body']['$id']; + + // make attributes + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'name'); + $this->assertEquals($response['body']['type'], 'string'); + $this->assertEquals($response['body']['size'], 256); + $this->assertEquals($response['body']['required'], true); + + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'age', + 'min' => 18, + 'max' => 65, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'age'); + $this->assertEquals($response['body']['type'], 'integer'); + $this->assertEquals($response['body']['min'], 18); + $this->assertEquals($response['body']['max'], 65); + $this->assertEquals($response['body']['required'], true); + + // make a bucket, upload a file to it! + // 1. enable compression, encryption + $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'gzip', + 'encryption' => true + ]); + $this->assertEquals(201, $bucketOne['headers']['status-code']); + $this->assertNotEmpty($bucketOne['body']['$id']); + + $bucketOneId = $bucketOne['body']['$id']; + + // 2. no compression and encryption + $bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket 2', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'none', + 'encryption' => false + ]); + + $this->assertNotEmpty($bucketTwo['body']['$id']); + $this->assertEquals(201, $bucketTwo['headers']['status-code']); + + $bucketTwoId = $bucketTwo['body']['$id']; + + $bucketIds = [ + 'compressed' => $bucketOneId, + 'uncompressed' => $bucketTwoId, + ]; + + $fileIds = []; + + foreach ($bucketIds as $label => $bucketId) { + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/documents.csv'), 'text/csv', 'documents.csv'), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals('documents.csv', $response['body']['name']); + $this->assertEquals('text/csv', $response['body']['mimeType']); + + $fileIds[$label] = $response['body']['$id']; + } + + // compressed, fail. + $compressed = $this->performCsvMigration( + [ + 'fileId' => $fileIds['compressed'], + 'bucketId' => $bucketIds['compressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + // fail on compressed, encrypted buckets! + $this->assertEquals(400, $compressed['body']['code']); + $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); + $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); + + // no compression, no encryption, pass. + $migration = $this->performCsvMigration( + [ + 'endpoint' => 'http://localhost/v1', + 'fileId' => $fileIds['uncompressed'], + 'bucketId' => $bucketIds['uncompressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('pending', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'migrationId' => $migration['body']['$id'], + ]; + } + + /** + * @depends testCreateCsvMigration + */ + public function testImportSuccessful(array $response): void + { + $databaseId = $response['databaseId']; + $collectionId = $response['collectionId']; + $migrationId = $response['migrationId']; + + $documentsCountInCSV = 100; + + // get migration stats + $this->assertEventually(function () use ($migrationId, $databaseId, $collectionId, $documentsCountInCSV) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $migration['body']['statusCounters']); + $this->assertEquals($documentsCountInCSV, $migration['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + }, 60000, 500); + + // get documents count + $documents = $this->client->call(Client::METHOD_GET, '/databases/'.$databaseId.'/collections/'.$collectionId.'/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + // there should be only 100! + Query::limit(150)->toString() + ] + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertIsArray($documents['body']['documents']); + $this->assertIsNumeric($documents['body']['total']); + $this->assertEquals($documentsCountInCSV, $documents['body']['total']); + } + + private function performCsvMigration(array $body): array + { + return $this->client->call(Client::METHOD_POST, '/migrations/csv', [ + 'content-type' => 'application/json', + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-appwrite-project' => $this->getProject()['$id'], + ], $body); + } } diff --git a/tests/resources/documents.csv b/tests/resources/documents.csv new file mode 100644 index 0000000000..ea1e33b5bd --- /dev/null +++ b/tests/resources/documents.csv @@ -0,0 +1,101 @@ +$id,name,age +hxfcwpcas5xokpwe,Diamond Mendez,56 +gw8nxwf6esn3tfwf,Michael Huff,20 +xb6bxg56lral1qy9,Alyssa Rodriguez,37 +imerjq5j36y3agh2,Barbara Smith,26 +07yq9qdlhmbzmr35,Evelyn Edwards,54 +ksqo631sbhwj5ltg,Tina Richardson,41 +j7zlndgu0gbshp15,Joel Hernandez,49 +mfntvnljrcmf7h6v,Zachary Cooper,59 +5f9b01nziqu2h8ed,Brittany Spears,20 +4vxzbnzraqznk5u8,Holly White,47 +d4ywy3mtphaatbpf,Kimberly Barnes,27 +88odnk6nthyyvbal,Stephen Miller,53 +08oekee3fn7mzaa5,Yvonne Newman,41 +quw55kn9895i5e4v,Carol Kane,38 +nge6bm8ykripei6f,Doris Foster,44 +4k16i33s0xl2ypx9,Joseph Stokes,28 +q0j5rxbgid66snyf,Steve Williams,31 +n1oxun7mqq3p103y,James Carey,29 +0dbvs840jkf8i0ye,Kathryn Henry,38 +5sfaidgs1h87v15v,Christopher Landry,23 +vg3punvfu5khmf41,Jennifer Mcgee,62 +f933qydr9u5b2r11,Cathy Church,35 +wjv87y1inf8yk32s,Jose Lopez,41 +uljysdvdlcyrbrwk,William Rose,30 +ot8xtzh77j55wq0s,Sarah Ford,26 +9t76vnsv2u36s43t,Alisha Jones,61 +66y4tnty62hw8c02,Kristin Kelly,61 +2punfblazi5v16ar,Brendan Stout,40 +sxhr4nf5w2gx4wbg,Kelly Cruz,18 +68dvrqfwqnkq5el9,Samantha Martin,50 +20192l6dbeinhkh0,David Santos,46 +si0l4dgay09ebfmf,Elizabeth Carroll,22 +lhse40vbldqb6ap1,Corey Owens,46 +h5t3pslykyx3kxfm,Shelby Mueller,65 +ldc0luydrw6jub0f,Dr. Sylvia Myers,29 +voc9628xg4dsgw2y,Scott Freeman,48 +o4y0gk3gqv1ax2fz,Christopher Atkinson,21 +u1n3x4e4u7e0vzj6,Sean Diaz,31 +s36eskwtm0w7lwr7,Bobby Dyer,57 +4hjnag1p5iwvtixd,Daniel Hall,62 +m91d80oxsa216zbh,Jennifer Ramirez,65 +5hj6858zo2g85n6v,Angela Jackson,57 +8m8oihv9a1e7nn92,Kelly Lewis,36 +7azy39la0no0mxi7,Jessica Munoz,55 +47pmjkhnnqhyit8c,Kelly George,65 +6j6cpy4kgneg1mmh,Anthony Johnson,65 +tnlmtvap1zz89km9,Regina Fields,61 +6cyuvnwwqdmrpfzh,Sharon Schaefer,30 +p1v4pyu2pqodc0ey,Jacob French,62 +6npynnhjt2jd05xo,Jessica Costa,23 +wcxedf13n2e9qi4l,George Hardy,53 +yf2xlcmszk2tqeig,Andrea Allison,20 +3bf2zzv7poststwa,Kevin Ferguson,32 +c2iataz0hhv39q63,Joseph Johnson,58 +3e8npxhov4a39pvq,Ashley Martinez,18 +t7dp41tysipytywq,Charles Nixon,23 +z8cztq7c47phyfhk,Carol Dudley,40 +2636f9d8r4ipm3h6,David Weber,51 +eh3f6wxtvkjq6ykq,Scott Robinson,32 +raskbwpsje69a59h,Anthony Hardy,38 +90hn1p0b4cs9e2og,Mackenzie Owens,52 +am3swwfbo076x0v1,Brian Foster,27 +5uw7utb9lq5cfncw,Hannah Forbes,56 +cs6mbfzkzifefx6r,Lauren Reed,26 +ftw3uvztziiz9x00,Morgan Smith,28 +uhrqseeo43mozpaq,Samantha Alexander,65 +pvvmzyfc1lxor11e,Tiffany Roberts,20 +jia7bdag4abz123s,Emily Hayes,34 +h6oozcngbz8o5x4y,Rebecca Villegas,52 +9v6z1pn2f9twcy12,Donald Shah,61 +wzz3jduioso77o7f,Denise Cain,59 +u51plhgvjodkswnr,Kristine Ramirez,53 +t1uhkmiytfyc13vc,Stacey Adkins,61 +iqaqnf0ybg2ct507,Daniel Hunt,20 +idwrwv2uu4hcpv2i,Roberta Johnson,48 +2yd2hd6auetjacyo,Jason Williamson,39 +egrmdbibnjhi914x,Sandra Robinson,50 +15m1pz2bb0ercgyk,Steve Rice,25 +0i21bhkxdagjurb7,Kimberly Fritz,53 +726ofi7h5snreq67,Brianna Reynolds,33 +csqxse3wym56eim6,Alexander Williams,50 +qeaoylnrsf8p3byg,Andrew Thomas,25 +edsswobumzyzbvhf,Austin Williams,57 +hdzhzpt0ahy5hkib,Nicholas Williams,24 +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48 +3z3o73x7adyuo6w0,Stacey Smith,39 +sse2u5zlgoqrgmcf,Laura Beck,20 +rvovijmvch58r4yx,Molly Clark,51 +doe06nrx8sg5mcuv,Carmen Morris,41 +jbjdwuvj5s4kw04y,Amanda Munoz,20 +6k2ewkla7js0yw23,Rachel Collins,44 +fcxuyr4kkhrnigu1,John Alexander,18 +d25fuwlos5mk07o0,Stacy Hunter,22 +1vdai2rxmwd57oet,Eric Massey,40 +pq4jnt9izu1wlrzd,Scott Garcia,20 +lz9kfc0lty5xcz14,Cassandra Nelson,35 +pu7w6tyab5jd4we9,Aaron Johnson,50 +8dupswd2kqwdyn8v,Shannon Sherman,45 +ye466l71jthiz2p6,April Garcia,60 +xogsmfwb73l16qdt,Evan Lynn,20 From cc7d038c3efae8b213649a0c109f3ff29856819b Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 08:54:50 +0530 Subject: [PATCH 13/20] address comments: change `deviceForCsvImports` to `deviceForImports`. --- app/controllers/api/migrations.php | 10 +++++----- app/init/resources.php | 2 +- app/worker.php | 2 +- src/Appwrite/Platform/Workers/Migrations.php | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 7dd8fe42f5..2c4090bed1 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -322,10 +322,10 @@ App::post('/v1/migrations/csv') ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('deviceForCsvImports') + ->inject('deviceForImports') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForCsvImports, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForImports, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -351,12 +351,12 @@ App::post('/v1/migrations/csv') // copy to temporary folder $migrationId = ID::unique(); - $newPath = $deviceForCsvImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); - if (!$deviceForFiles->transfer($path, $newPath, $deviceForCsvImports)) { + $newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); + if (!$deviceForFiles->transfer($path, $newPath, $deviceForImports)) { throw new \Exception("Unable to copy file"); } - $fileSize = $deviceForCsvImports->getFileSize($path); + $fileSize = $deviceForImports->getFileSize($path); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); $migration = $dbForProject->createDocument('migrations', new Document([ diff --git a/app/init/resources.php b/app/init/resources.php index 6aceaf6fb4..ed124be1ac 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -508,7 +508,7 @@ App::setResource('deviceForFiles', function ($project) { return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); -App::setResource('deviceForCsvImports', function (Document $project) { +App::setResource('deviceForImports', function (Document $project) { return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/worker.php b/app/worker.php index cd3b433d44..6a62fb7e7d 100644 --- a/app/worker.php +++ b/app/worker.php @@ -339,7 +339,7 @@ Server::setResource('pools', function (Registry $register) { return $register->get('pools'); }, ['register']); -Server::setResource('deviceForCsvImports', function (Document $project) { +Server::setResource('deviceForImports', function (Document $project) { return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index c4182f8deb..323924bc86 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -34,7 +34,7 @@ class Migrations extends Action protected Database $dbForPlatform; - protected Device $deviceForCsvImports; + protected Device $deviceForImports; protected Document $project; @@ -61,17 +61,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') - ->inject('deviceForCsvImports') - ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime, $deviceForCsvImports)); + ->inject('deviceForImports') + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime, $deviceForImports)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void { $payload = $message->getPayload() ?? []; - $this->deviceForCsvImports = $deviceForCsvImports; + $this->deviceForImports = $deviceForImports; if (empty($payload)) { throw new Exception('Missing payload'); @@ -139,7 +139,7 @@ class Migrations extends Action CSV::getName() => new CSV( $resourceId, $migrationOptions['path'], - $this->deviceForCsvImports, + $this->deviceForImports, $this->dbForProject ), default => throw new \Exception('Invalid source type'), From 680b214950e804c188fc49f550fe7a48e13fc97f Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 08:59:30 +0530 Subject: [PATCH 14/20] update: storage volume name. --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 13a4136ca6..6de5ad4cd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw - - appwrite-csv-imports:/storage/imports:rw + - appwrite-imports:/storage/imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -674,7 +674,7 @@ services: - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests # for csv import access - - appwrite-csv-imports:/storage/imports:rw + - appwrite-imports:/storage/imports:rw depends_on: - mariadb environment: @@ -1135,7 +1135,7 @@ volumes: appwrite-redis: appwrite-cache: appwrite-uploads: - appwrite-csv-imports: + appwrite-imports: appwrite-certificates: appwrite-functions: appwrite-builds: From 36ab6ce236cb5ad0f5d4f470816591adfcf4114b Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 10:36:05 +0530 Subject: [PATCH 15/20] address comments: add more tests. --- .../Services/Migrations/MigrationsBase.php | 110 +++++++++++++++++- tests/resources/{ => csv}/documents.csv | 0 tests/resources/csv/irrelevant-column.csv | 101 ++++++++++++++++ tests/resources/csv/missing-column.csv | 101 ++++++++++++++++ tests/resources/csv/missing-row.csv | 101 ++++++++++++++++ 5 files changed, 410 insertions(+), 3 deletions(-) rename tests/resources/{ => csv}/documents.csv (100%) create mode 100644 tests/resources/csv/irrelevant-column.csv create mode 100644 tests/resources/csv/missing-column.csv create mode 100644 tests/resources/csv/missing-row.csv diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index f7e7d20e99..45b57d6b0c 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1011,23 +1011,40 @@ trait MigrationsBase $bucketIds = [ 'compressed' => $bucketOneId, 'uncompressed' => $bucketTwoId, + + // in uncompressed buckets! + 'missing-row' => $bucketTwoId, + 'missing-column' => $bucketTwoId, + 'irrelevant-column' => $bucketTwoId, ]; $fileIds = []; foreach ($bucketIds as $label => $bucketId) { + $csvFileName = match ($label) { + 'missing-row', + 'missing-column', + 'irrelevant-column' => "{$label}.csv", + default => 'documents.csv', + }; + + $mimeType = match ($csvFileName) { + default => 'text/csv', + 'missing-row.csv' => 'text/plain', // invalid csv structure, falls back to plain text! + }; + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ 'content-type' => 'multipart/form-data', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'fileId' => ID::unique(), - 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/documents.csv'), 'text/csv', 'documents.csv'), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/'.$csvFileName), $mimeType, $csvFileName), ]); $this->assertEquals(201, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); - $this->assertEquals('documents.csv', $response['body']['name']); - $this->assertEquals('text/csv', $response['body']['mimeType']); + $this->assertEquals($csvFileName, $response['body']['name']); + $this->assertEquals($mimeType, $response['body']['mimeType']); $fileIds[$label] = $response['body']['$id']; } @@ -1046,6 +1063,93 @@ trait MigrationsBase $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); + // missing attribute, fail in worker. + $missingColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['missing-column'], + 'bucketId' => $bucketIds['missing-column'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($missingColumn, $databaseId, $collectionId) { + $migrationId = $missingColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains("CSV header mismatch. Missing attribute: 'age'") + ); + }, 60000, 500); + + // missing row data, fail in worker. + $missingColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['missing-row'], + 'bucketId' => $bucketIds['missing-row'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($missingColumn, $databaseId, $collectionId) { + $migrationId = $missingColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains('CSV row does not match the number of header columns') + ); + }, 60000, 500); + + // irrelevant column - email, fail in worker. + $irrelevantColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['irrelevant-column'], + 'bucketId' => $bucketIds['irrelevant-column'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($irrelevantColumn, $databaseId, $collectionId) { + $migrationId = $irrelevantColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains("CSV header mismatch. Unexpected attribute: 'email'") + ); + }, 60000, 500); + // no compression, no encryption, pass. $migration = $this->performCsvMigration( [ diff --git a/tests/resources/documents.csv b/tests/resources/csv/documents.csv similarity index 100% rename from tests/resources/documents.csv rename to tests/resources/csv/documents.csv diff --git a/tests/resources/csv/irrelevant-column.csv b/tests/resources/csv/irrelevant-column.csv new file mode 100644 index 0000000000..92105ceaa2 --- /dev/null +++ b/tests/resources/csv/irrelevant-column.csv @@ -0,0 +1,101 @@ +$id,name,age,email +hxfcwpcas5xokpwe,Diamond Mendez,56,diamond.mendez@example.com +gw8nxwf6esn3tfwf,Michael Huff,20,michael.huff@example.com +xb6bxg56lral1qy9,Alyssa Rodriguez,37,alyssa.rodriguez@example.com +imerjq5j36y3agh2,Barbara Smith,26,barbara.smith@example.com +07yq9qdlhmbzmr35,Evelyn Edwards,54,evelyn.edwards@example.com +ksqo631sbhwj5ltg,Tina Richardson,41,tina.richardson@example.com +j7zlndgu0gbshp15,Joel Hernandez,49,joel.hernandez@example.com +mfntvnljrcmf7h6v,Zachary Cooper,59,zachary.cooper@example.com +5f9b01nziqu2h8ed,Brittany Spears,20,brittany.spears@example.com +4vxzbnzraqznk5u8,Holly White,47,holly.white@example.com +d4ywy3mtphaatbpf,Kimberly Barnes,27,kimberly.barnes@example.com +88odnk6nthyyvbal,Stephen Miller,53,stephen.miller@example.com +08oekee3fn7mzaa5,Yvonne Newman,41,yvonne.newman@example.com +quw55kn9895i5e4v,Carol Kane,38,carol.kane@example.com +nge6bm8ykripei6f,Doris Foster,44,doris.foster@example.com +4k16i33s0xl2ypx9,Joseph Stokes,28,joseph.stokes@example.com +q0j5rxbgid66snyf,Steve Williams,31,steve.williams@example.com +n1oxun7mqq3p103y,James Carey,29,james.carey@example.com +0dbvs840jkf8i0ye,Kathryn Henry,38,kathryn.henry@example.com +5sfaidgs1h87v15v,Christopher Landry,23,christopher.landry@example.com +vg3punvfu5khmf41,Jennifer Mcgee,62,jennifer.mcgee@example.com +f933qydr9u5b2r11,Cathy Church,35,cathy.church@example.com +wjv87y1inf8yk32s,Jose Lopez,41,jose.lopez@example.com +uljysdvdlcyrbrwk,William Rose,30,william.rose@example.com +ot8xtzh77j55wq0s,Sarah Ford,26,sarah.ford@example.com +9t76vnsv2u36s43t,Alisha Jones,61,alisha.jones@example.com +66y4tnty62hw8c02,Kristin Kelly,61,kristin.kelly@example.com +2punfblazi5v16ar,Brendan Stout,40,brendan.stout@example.com +sxhr4nf5w2gx4wbg,Kelly Cruz,18,kelly.cruz@example.com +68dvrqfwqnkq5el9,Samantha Martin,50,samantha.martin@example.com +20192l6dbeinhkh0,David Santos,46,david.santos@example.com +si0l4dgay09ebfmf,Elizabeth Carroll,22,elizabeth.carroll@example.com +lhse40vbldqb6ap1,Corey Owens,46,corey.owens@example.com +h5t3pslykyx3kxfm,Shelby Mueller,65,shelby.mueller@example.com +ldc0luydrw6jub0f,Dr. Sylvia Myers,29,sylvia.myers@example.com +voc9628xg4dsgw2y,Scott Freeman,48,scott.freeman@example.com +o4y0gk3gqv1ax2fz,Christopher Atkinson,21,christopher.atkinson@example.com +u1n3x4e4u7e0vzj6,Sean Diaz,31,sean.diaz@example.com +s36eskwtm0w7lwr7,Bobby Dyer,57,bobby.dyer@example.com +4hjnag1p5iwvtixd,Daniel Hall,62,daniel.hall@example.com +m91d80oxsa216zbh,Jennifer Ramirez,65,jennifer.ramirez@example.com +5hj6858zo2g85n6v,Angela Jackson,57,angela.jackson@example.com +8m8oihv9a1e7nn92,Kelly Lewis,36,kelly.lewis@example.com +7azy39la0no0mxi7,Jessica Munoz,55,jessica.munoz@example.com +47pmjkhnnqhyit8c,Kelly George,65,kelly.george@example.com +6j6cpy4kgneg1mmh,Anthony Johnson,65,anthony.johnson@example.com +tnlmtvap1zz89km9,Regina Fields,61,regina.fields@example.com +6cyuvnwwqdmrpfzh,Sharon Schaefer,30,sharon.schaefer@example.com +p1v4pyu2pqodc0ey,Jacob French,62,jacob.french@example.com +6npynnhjt2jd05xo,Jessica Costa,23,jessica.costa@example.com +wcxedf13n2e9qi4l,George Hardy,53,george.hardy@example.com +yf2xlcmszk2tqeig,Andrea Allison,20,andrea.allison@example.com +3bf2zzv7poststwa,Kevin Ferguson,32,kevin.ferguson@example.com +c2iataz0hhv39q63,Joseph Johnson,58,joseph.johnson@example.com +3e8npxhov4a39pvq,Ashley Martinez,18,ashley.martinez@example.com +t7dp41tysipytywq,Charles Nixon,23,charles.nixon@example.com +z8cztq7c47phyfhk,Carol Dudley,40,carol.dudley@example.com +2636f9d8r4ipm3h6,David Weber,51,david.weber@example.com +eh3f6wxtvkjq6ykq,Scott Robinson,32,scott.robinson@example.com +raskbwpsje69a59h,Anthony Hardy,38,anthony.hardy@example.com +90hn1p0b4cs9e2og,Mackenzie Owens,52,mackenzie.owens@example.com +am3swwfbo076x0v1,Brian Foster,27,brian.foster@example.com +5uw7utb9lq5cfncw,Hannah Forbes,56,hannah.forbes@example.com +cs6mbfzkzifefx6r,Lauren Reed,26,lauren.reed@example.com +ftw3uvztziiz9x00,Morgan Smith,28,morgan.smith@example.com +uhrqseeo43mozpaq,Samantha Alexander,65,samantha.alexander@example.com +pvvmzyfc1lxor11e,Tiffany Roberts,20,tiffany.roberts@example.com +jia7bdag4abz123s,Emily Hayes,34,emily.hayes@example.com +h6oozcngbz8o5x4y,Rebecca Villegas,52,rebecca.villegas@example.com +9v6z1pn2f9twcy12,Donald Shah,61,donald.shah@example.com +wzz3jduioso77o7f,Denise Cain,59,denise.cain@example.com +u51plhgvjodkswnr,Kristine Ramirez,53,kristine.ramirez@example.com +t1uhkmiytfyc13vc,Stacey Adkins,61,stacey.adkins@example.com +iqaqnf0ybg2ct507,Daniel Hunt,20,daniel.hunt@example.com +idwrwv2uu4hcpv2i,Roberta Johnson,48,roberta.johnson@example.com +2yd2hd6auetjacyo,Jason Williamson,39,jason.williamson@example.com +egrmdbibnjhi914x,Sandra Robinson,50,sandra.robinson@example.com +15m1pz2bb0ercgyk,Steve Rice,25,steve.rice@example.com +0i21bhkxdagjurb7,Kimberly Fritz,53,kimberly.fritz@example.com +726ofi7h5snreq67,Brianna Reynolds,33,brianna.reynolds@example.com +csqxse3wym56eim6,Alexander Williams,50,alexander.williams@example.com +qeaoylnrsf8p3byg,Andrew Thomas,25,andrew.thomas@example.com +edsswobumzyzbvhf,Austin Williams,57,austin.williams@example.com +hdzhzpt0ahy5hkib,Nicholas Williams,24,nicholas.williams@example.com +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48,michelle.cisneros@example.com +3z3o73x7adyuo6w0,Stacey Smith,39,stacey.smith@example.com +sse2u5zlgoqrgmcf,Laura Beck,20,laura.beck@example.com +rvovijmvch58r4yx,Molly Clark,51,molly.clark@example.com +doe06nrx8sg5mcuv,Carmen Morris,41,carmen.morris@example.com +jbjdwuvj5s4kw04y,Amanda Munoz,20,amanda.munoz@example.com +6k2ewkla7js0yw23,Rachel Collins,44,rachel.collins@example.com +fcxuyr4kkhrnigu1,John Alexander,18,john.alexander@example.com +d25fuwlos5mk07o0,Stacy Hunter,22,stacy.hunter@example.com +1vdai2rxmwd57oet,Eric Massey,40,eric.massey@example.com +pq4jnt9izu1wlrzd,Scott Garcia,20,scott.garcia@example.com +lz9kfc0lty5xcz14,Cassandra Nelson,35,cassandra.nelson@example.com +pu7w6tyab5jd4we9,Aaron Johnson,50,aaron.johnson@example.com +8dupswd2kqwdyn8v,Shannon Sherman,45,shannon.sherman@example.com +ye466l71jthiz2p6,April Garcia,60,april.garcia@example.com +xogsmfwb73l16qdt,Evan Lynn,20,evan.lynn@example.com diff --git a/tests/resources/csv/missing-column.csv b/tests/resources/csv/missing-column.csv new file mode 100644 index 0000000000..e57b5ccb2e --- /dev/null +++ b/tests/resources/csv/missing-column.csv @@ -0,0 +1,101 @@ +$id,name +hxfcwpcas5xokpwe,Diamond Mendez +gw8nxwf6esn3tfwf,Michael Huff +xb6bxg56lral1qy9,Alyssa Rodriguez +imerjq5j36y3agh2,Barbara Smith +07yq9qdlhmbzmr35,Evelyn Edwards +ksqo631sbhwj5ltg,Tina Richardson +j7zlndgu0gbshp15,Joel Hernandez +mfntvnljrcmf7h6v,Zachary Cooper +5f9b01nziqu2h8ed,Brittany Spears +4vxzbnzraqznk5u8,Holly White +d4ywy3mtphaatbpf,Kimberly Barnes +88odnk6nthyyvbal,Stephen Miller +08oekee3fn7mzaa5,Yvonne Newman +quw55kn9895i5e4v,Carol Kane +nge6bm8ykripei6f,Doris Foster +4k16i33s0xl2ypx9,Joseph Stokes +q0j5rxbgid66snyf,Steve Williams +n1oxun7mqq3p103y,James Carey +0dbvs840jkf8i0ye,Kathryn Henry +5sfaidgs1h87v15v,Christopher Landry +vg3punvfu5khmf41,Jennifer Mcgee +f933qydr9u5b2r11,Cathy Church +wjv87y1inf8yk32s,Jose Lopez +uljysdvdlcyrbrwk,William Rose +ot8xtzh77j55wq0s,Sarah Ford +9t76vnsv2u36s43t,Alisha Jones +66y4tnty62hw8c02,Kristin Kelly +2punfblazi5v16ar,Brendan Stout +sxhr4nf5w2gx4wbg,Kelly Cruz +68dvrqfwqnkq5el9,Samantha Martin +20192l6dbeinhkh0,David Santos +si0l4dgay09ebfmf,Elizabeth Carroll +lhse40vbldqb6ap1,Corey Owens +h5t3pslykyx3kxfm,Shelby Mueller +ldc0luydrw6jub0f,Dr. Sylvia Myers +voc9628xg4dsgw2y,Scott Freeman +o4y0gk3gqv1ax2fz,Christopher Atkinson +u1n3x4e4u7e0vzj6,Sean Diaz +s36eskwtm0w7lwr7,Bobby Dyer +4hjnag1p5iwvtixd,Daniel Hall +m91d80oxsa216zbh,Jennifer Ramirez +5hj6858zo2g85n6v,Angela Jackson +8m8oihv9a1e7nn92,Kelly Lewis +7azy39la0no0mxi7,Jessica Munoz +47pmjkhnnqhyit8c,Kelly George +6j6cpy4kgneg1mmh,Anthony Johnson +tnlmtvap1zz89km9,Regina Fields +6cyuvnwwqdmrpfzh,Sharon Schaefer +p1v4pyu2pqodc0ey,Jacob French +6npynnhjt2jd05xo,Jessica Costa +wcxedf13n2e9qi4l,George Hardy +yf2xlcmszk2tqeig,Andrea Allison +3bf2zzv7poststwa,Kevin Ferguson +c2iataz0hhv39q63,Joseph Johnson +3e8npxhov4a39pvq,Ashley Martinez +t7dp41tysipytywq,Charles Nixon +z8cztq7c47phyfhk,Carol Dudley +2636f9d8r4ipm3h6,David Weber +eh3f6wxtvkjq6ykq,Scott Robinson +raskbwpsje69a59h,Anthony Hardy +90hn1p0b4cs9e2og,Mackenzie Owens +am3swwfbo076x0v1,Brian Foster +5uw7utb9lq5cfncw,Hannah Forbes +cs6mbfzkzifefx6r,Lauren Reed +ftw3uvztziiz9x00,Morgan Smith +uhrqseeo43mozpaq,Samantha Alexander +pvvmzyfc1lxor11e,Tiffany Roberts +jia7bdag4abz123s,Emily Hayes +h6oozcngbz8o5x4y,Rebecca Villegas +9v6z1pn2f9twcy12,Donald Shah +wzz3jduioso77o7f,Denise Cain +u51plhgvjodkswnr,Kristine Ramirez +t1uhkmiytfyc13vc,Stacey Adkins +iqaqnf0ybg2ct507,Daniel Hunt +idwrwv2uu4hcpv2i,Roberta Johnson +2yd2hd6auetjacyo,Jason Williamson +egrmdbibnjhi914x,Sandra Robinson +15m1pz2bb0ercgyk,Steve Rice +0i21bhkxdagjurb7,Kimberly Fritz +726ofi7h5snreq67,Brianna Reynolds +csqxse3wym56eim6,Alexander Williams +qeaoylnrsf8p3byg,Andrew Thomas +edsswobumzyzbvhf,Austin Williams +hdzhzpt0ahy5hkib,Nicholas Williams +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros +3z3o73x7adyuo6w0,Stacey Smith +sse2u5zlgoqrgmcf,Laura Beck +rvovijmvch58r4yx,Molly Clark +doe06nrx8sg5mcuv,Carmen Morris +jbjdwuvj5s4kw04y,Amanda Munoz +6k2ewkla7js0yw23,Rachel Collins +fcxuyr4kkhrnigu1,John Alexander +d25fuwlos5mk07o0,Stacy Hunter +1vdai2rxmwd57oet,Eric Massey +pq4jnt9izu1wlrzd,Scott Garcia +lz9kfc0lty5xcz14,Cassandra Nelson +pu7w6tyab5jd4we9,Aaron Johnson +8dupswd2kqwdyn8v,Shannon Sherman +ye466l71jthiz2p6,April Garcia +xogsmfwb73l16qdt,Evan Lynn diff --git a/tests/resources/csv/missing-row.csv b/tests/resources/csv/missing-row.csv new file mode 100644 index 0000000000..7399fa9f51 --- /dev/null +++ b/tests/resources/csv/missing-row.csv @@ -0,0 +1,101 @@ +$id,name,age +hxfcwpcas5xokpwe,Diamond Mendez +gw8nxwf6esn3tfwf,Michael Huff +xb6bxg56lral1qy9,Alyssa Rodriguez +imerjq5j36y3agh2,Barbara Smith +07yq9qdlhmbzmr35,Evelyn Edwards +ksqo631sbhwj5ltg,Tina Richardson +j7zlndgu0gbshp15,Joel Hernandez +mfntvnljrcmf7h6v,Zachary Cooper +5f9b01nziqu2h8ed,Brittany Spears +4vxzbnzraqznk5u8,Holly White +d4ywy3mtphaatbpf,Kimberly Barnes +88odnk6nthyyvbal,Stephen Miller +08oekee3fn7mzaa5,Yvonne Newman +quw55kn9895i5e4v,Carol Kane +nge6bm8ykripei6f,Doris Foster +4k16i33s0xl2ypx9,Joseph Stokes +q0j5rxbgid66snyf,Steve Williams +n1oxun7mqq3p103y,James Carey +0dbvs840jkf8i0ye,Kathryn Henry +5sfaidgs1h87v15v,Christopher Landry +vg3punvfu5khmf41,Jennifer Mcgee +f933qydr9u5b2r11,Cathy Church +wjv87y1inf8yk32s,Jose Lopez +uljysdvdlcyrbrwk,William Rose +ot8xtzh77j55wq0s,Sarah Ford +9t76vnsv2u36s43t,Alisha Jones +66y4tnty62hw8c02,Kristin Kelly +2punfblazi5v16ar,Brendan Stout +sxhr4nf5w2gx4wbg,Kelly Cruz +68dvrqfwqnkq5el9,Samantha Martin +20192l6dbeinhkh0,David Santos +si0l4dgay09ebfmf,Elizabeth Carroll +lhse40vbldqb6ap1,Corey Owens +h5t3pslykyx3kxfm,Shelby Mueller +ldc0luydrw6jub0f,Dr. Sylvia Myers +voc9628xg4dsgw2y,Scott Freeman +o4y0gk3gqv1ax2fz,Christopher Atkinson +u1n3x4e4u7e0vzj6,Sean Diaz +s36eskwtm0w7lwr7,Bobby Dyer +4hjnag1p5iwvtixd,Daniel Hall +m91d80oxsa216zbh,Jennifer Ramirez +5hj6858zo2g85n6v,Angela Jackson +8m8oihv9a1e7nn92,Kelly Lewis +7azy39la0no0mxi7,Jessica Munoz +47pmjkhnnqhyit8c,Kelly George +6j6cpy4kgneg1mmh,Anthony Johnson +tnlmtvap1zz89km9,Regina Fields +6cyuvnwwqdmrpfzh,Sharon Schaefer +p1v4pyu2pqodc0ey,Jacob French +6npynnhjt2jd05xo,Jessica Costa +wcxedf13n2e9qi4l,George Hardy +yf2xlcmszk2tqeig,Andrea Allison +3bf2zzv7poststwa,Kevin Ferguson +c2iataz0hhv39q63,Joseph Johnson +3e8npxhov4a39pvq,Ashley Martinez +t7dp41tysipytywq,Charles Nixon +z8cztq7c47phyfhk,Carol Dudley +2636f9d8r4ipm3h6,David Weber +eh3f6wxtvkjq6ykq,Scott Robinson +raskbwpsje69a59h,Anthony Hardy +90hn1p0b4cs9e2og,Mackenzie Owens +am3swwfbo076x0v1,Brian Foster +5uw7utb9lq5cfncw,Hannah Forbes +cs6mbfzkzifefx6r,Lauren Reed +ftw3uvztziiz9x00,Morgan Smith +uhrqseeo43mozpaq,Samantha Alexander +pvvmzyfc1lxor11e,Tiffany Roberts +jia7bdag4abz123s,Emily Hayes +h6oozcngbz8o5x4y,Rebecca Villegas +9v6z1pn2f9twcy12,Donald Shah +wzz3jduioso77o7f,Denise Cain +u51plhgvjodkswnr,Kristine Ramirez +t1uhkmiytfyc13vc,Stacey Adkins +iqaqnf0ybg2ct507,Daniel Hunt +idwrwv2uu4hcpv2i,Roberta Johnson +2yd2hd6auetjacyo,Jason Williamson +egrmdbibnjhi914x,Sandra Robinson +15m1pz2bb0ercgyk,Steve Rice +0i21bhkxdagjurb7,Kimberly Fritz +726ofi7h5snreq67,Brianna Reynolds +csqxse3wym56eim6,Alexander Williams +qeaoylnrsf8p3byg,Andrew Thomas +edsswobumzyzbvhf,Austin Williams +hdzhzpt0ahy5hkib,Nicholas Williams +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros +3z3o73x7adyuo6w0,Stacey Smith +sse2u5zlgoqrgmcf,Laura Beck +rvovijmvch58r4yx,Molly Clark +doe06nrx8sg5mcuv,Carmen Morris +jbjdwuvj5s4kw04y,Amanda Munoz +6k2ewkla7js0yw23,Rachel Collins +fcxuyr4kkhrnigu1,John Alexander +d25fuwlos5mk07o0,Stacy Hunter +1vdai2rxmwd57oet,Eric Massey +pq4jnt9izu1wlrzd,Scott Garcia +lz9kfc0lty5xcz14,Cassandra Nelson +pu7w6tyab5jd4we9,Aaron Johnson +8dupswd2kqwdyn8v,Shannon Sherman +ye466l71jthiz2p6,April Garcia +xogsmfwb73l16qdt,Evan Lynn From fda0af6694a0c62b1ba4bc02434867f91c5ca9ef Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 10:51:30 +0530 Subject: [PATCH 16/20] bump: dependency. --- composer.json | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index d0f3803210..39477c83a0 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.9.0", + "utopia-php/migration": "0.9.1", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", diff --git a/composer.lock b/composer.lock index 46353a4a11..7fa497a8f5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "85afadfc660334537aaba2c355f98b9c", + "content-hash": "e22cfd0e495f55633218f029d8c7ec9d", "packages": [ { "name": "adhocore/jwt", @@ -3951,16 +3951,16 @@ }, { "name": "utopia-php/migration", - "version": "0.9.0", + "version": "0.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "545705e251b766940d2833893f267975d73abe32" + "reference": "f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/545705e251b766940d2833893f267975d73abe32", - "reference": "545705e251b766940d2833893f267975d73abe32", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3", + "reference": "f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3", "shasum": "" }, "require": { @@ -4001,9 +4001,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.9.0" + "source": "https://github.com/utopia-php/migration/tree/0.9.1" }, - "time": "2025-04-16T07:52:53+00:00" + "time": "2025-04-17T05:18:58+00:00" }, { "name": "utopia-php/orchestration", From 767e202b342a2fdca91e08da22029d30e160117a Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 10:54:23 +0530 Subject: [PATCH 17/20] update: name. --- app/controllers/api/migrations.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 2c4090bed1..4a1e5de227 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -364,7 +364,7 @@ App::post('/v1/migrations/csv') 'status' => 'pending', 'stage' => 'init', 'source' => CSV::getName(), - 'destination' => Appwrite::class::getName(), + 'destination' => Appwrite::getName(), 'resources' => $resources, 'resourceId' => $resourceId, 'resourceType' => Resource::TYPE_DATABASE, From 6d3e40d73b1c378598aac4a58ce77afbb54d3e15 Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 11:03:03 +0530 Subject: [PATCH 18/20] address comments: change order, remove comment. --- docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6de5ad4cd0..9bd55becd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -670,11 +670,10 @@ services: networks: - appwrite volumes: + - appwrite-imports:/storage/imports:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests - # for csv import access - - appwrite-imports:/storage/imports:rw depends_on: - mariadb environment: From 9f69aa624e928b560e447bedcc9db6c7cea9d52a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 17 Apr 2025 12:11:52 +0200 Subject: [PATCH 19/20] Update comments --- app/config/collections/projects.php | 4 ++-- app/config/template-runtimes.php | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 806b6cf088..8362d1b9fd 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1202,7 +1202,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('adapter'), // ssr or static + '$id' => ID::custom('adapter'), // ssr or static; named this way as it's a term in SSR frameworks 'type' => Database::VAR_STRING, 'format' => '', 'size' => 128, @@ -1727,7 +1727,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('adapter'), + '$id' => ID::custom('adapter'), // ssr or static; named this way as it's a term in SSR frameworks 'type' => Database::VAR_STRING, 'format' => '', 'size' => 128, diff --git a/app/config/template-runtimes.php b/app/config/template-runtimes.php index 4a2436b2b8..8f1c0198c2 100644 --- a/app/config/template-runtimes.php +++ b/app/config/template-runtimes.php @@ -1,5 +1,8 @@ [ 'name' => 'node', From db4459841958105b4e3aa4292dbb2970596f3846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 17 Apr 2025 12:16:58 +0200 Subject: [PATCH 20/20] Update composer.lock --- composer.lock | 233 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 168 insertions(+), 65 deletions(-) diff --git a/composer.lock b/composer.lock index 7fa497a8f5..fea7bf6d51 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e22cfd0e495f55633218f029d8c7ec9d", + "content-hash": "e7875026636ccec909f9aa4d79091d5b", "packages": [ { "name": "adhocore/jwt", @@ -157,16 +157,16 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.16.5", + "version": "0.19.0", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9" + "reference": "8d21483efc19b9d977e323188989ee67a188464b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/runtimes/zipball/1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", - "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", + "url": "https://api.github.com/repos/appwrite/runtimes/zipball/8d21483efc19b9d977e323188989ee67a188464b", + "reference": "8d21483efc19b9d977e323188989ee67a188464b", "shasum": "" }, "require": { @@ -206,9 +206,9 @@ ], "support": { "issues": "https://github.com/appwrite/runtimes/issues", - "source": "https://github.com/appwrite/runtimes/tree/0.16.5" + "source": "https://github.com/appwrite/runtimes/tree/0.19.0" }, - "time": "2024-11-25T15:17:06+00:00" + "time": "2025-03-25T22:37:51+00:00" }, { "name": "beberlei/assert", @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.3", + "version": "1.2.4", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc" + "reference": "47fcb66ae5328c5a799195247b1dce551d85873e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/0e7804c176c4b09d95b7985400aa38ce544cb7fc", - "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/47fcb66ae5328c5a799195247b1dce551d85873e", + "reference": "47fcb66ae5328c5a799195247b1dce551d85873e", "shasum": "" }, "require": { @@ -1451,7 +1451,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-04-08T09:55:41+00:00" + "time": "2025-04-15T07:02:07+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3351,16 +3351,16 @@ }, { "name": "utopia-php/cli", - "version": "0.15.1", + "version": "0.15.2", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65" + "reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/d69bbe51a6a94dc4e5bcdd542b5938038b985a65", - "reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/da00ff6b8b29a826a1794002ae43442cdf3a0f5f", + "reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f", "shasum": "" }, "require": { @@ -3394,9 +3394,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.15.1" + "source": "https://github.com/utopia-php/cli/tree/0.15.2" }, - "time": "2024-10-04T13:55:36+00:00" + "time": "2025-04-15T10:08:48+00:00" }, { "name": "utopia-php/compression", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.64.1", + "version": "0.64.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b" + "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", - "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", + "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", "shasum": "" }, "require": { @@ -3547,9 +3547,54 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.64.1" + "source": "https://github.com/utopia-php/database/tree/0.64.2" }, - "time": "2025-04-02T00:35:29+00:00" + "time": "2025-04-09T07:53:05+00:00" + }, + { + "name": "utopia-php/detector", + "version": "0.1.4", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/detector.git", + "reference": "895a4147463965b5f9cbc083b764b6476f547879" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/895a4147463965b5f9cbc083b764b6476f547879", + "reference": "895a4147463965b5f9cbc083b764b6476f547879", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.8.*", + "phpunit/phpunit": "^9.4" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Detector\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library for fast and reliable environment identification.", + "keywords": [ + "detector", + "framework", + "php", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/detector/issues", + "source": "https://github.com/utopia-php/detector/tree/0.1.4" + }, + "time": "2025-04-09T11:50:45+00:00" }, { "name": "utopia-php/domains", @@ -3660,16 +3705,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.4.0", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e" + "reference": "65095dac14037db0c822fb5e209e5bd3187a0303" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/46e791ff6a95864517750b9df6bbf4a17e3c9c4e", - "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/65095dac14037db0c822fb5e209e5bd3187a0303", + "reference": "65095dac14037db0c822fb5e209e5bd3187a0303", "shasum": "" }, "require": { @@ -3693,9 +3738,9 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.4.0" + "source": "https://github.com/utopia-php/fetch/tree/0.4.1" }, - "time": "2025-03-11T21:06:56+00:00" + "time": "2025-04-14T07:34:27+00:00" }, { "name": "utopia-php/framework", @@ -3746,16 +3791,16 @@ }, { "name": "utopia-php/image", - "version": "0.8.1", + "version": "0.8.2", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "e8cc7dd14f423270a1b7570ec0dae88a66195b63" + "reference": "6c736965177f9a9e71311e22b80cfa88511768e9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/e8cc7dd14f423270a1b7570ec0dae88a66195b63", - "reference": "e8cc7dd14f423270a1b7570ec0dae88a66195b63", + "url": "https://api.github.com/repos/utopia-php/image/zipball/6c736965177f9a9e71311e22b80cfa88511768e9", + "reference": "6c736965177f9a9e71311e22b80cfa88511768e9", "shasum": "" }, "require": { @@ -3789,9 +3834,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.8.1" + "source": "https://github.com/utopia-php/image/tree/0.8.2" }, - "time": "2025-04-04T18:55:20+00:00" + "time": "2025-04-08T11:31:45+00:00" }, { "name": "utopia-php/locale", @@ -4107,16 +4152,16 @@ }, { "name": "utopia-php/pools", - "version": "0.8.0", + "version": "0.8.2", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba" + "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/60733929dc328e7ea47e800579c8bbf0d49df5ba", - "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", + "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", "shasum": "" }, "require": { @@ -4153,9 +4198,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.8.0" + "source": "https://github.com/utopia-php/pools/tree/0.8.2" }, - "time": "2025-03-19T10:22:03+00:00" + "time": "2025-04-17T02:04:54+00:00" }, { "name": "utopia-php/preloader", @@ -4543,16 +4588,16 @@ }, { "name": "utopia-php/vcs", - "version": "0.9.4", + "version": "0.10.1", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a" + "reference": "6be02650cc361764900ade8c129f309df263eb74" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/1a8d280b176acc99ea8d9e7364b8767cbb206b4a", - "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/6be02650cc361764900ade8c129f309df263eb74", + "reference": "6be02650cc361764900ade8c129f309df263eb74", "shasum": "" }, "require": { @@ -4570,8 +4615,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\VCS\\": "src/VCS", - "Utopia\\Detector\\": "src/Detector" + "Utopia\\VCS\\": "src/VCS" } }, "notification-url": "https://packagist.org/downloads/", @@ -4587,9 +4631,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/0.9.4" + "source": "https://github.com/utopia-php/vcs/tree/0.10.1" }, - "time": "2025-03-13T10:09:45+00:00" + "time": "2025-03-18T11:44:09+00:00" }, { "name": "utopia-php/websocket", @@ -4767,16 +4811,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.11", + "version": "0.40.12", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" + "reference": "182ec17848f81b78c336379bac94ff92b7a73365" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", - "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/182ec17848f81b78c336379bac94ff92b7a73365", + "reference": "182ec17848f81b78c336379bac94ff92b7a73365", "shasum": "" }, "require": { @@ -4812,9 +4856,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.12" }, - "time": "2025-03-26T10:53:16+00:00" + "time": "2025-04-02T23:36:11+00:00" }, { "name": "doctrine/annotations", @@ -5041,16 +5085,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.2", + "version": "v1.22.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" + "reference": "7ddfaa6523a675fae5c4123ee38fc6bfb8ee4f36" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", - "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", + "url": "https://api.github.com/repos/laravel/pint/zipball/7ddfaa6523a675fae5c4123ee38fc6bfb8ee4f36", + "reference": "7ddfaa6523a675fae5c4123ee38fc6bfb8ee4f36", "shasum": "" }, "require": { @@ -5061,9 +5105,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.72.0", + "friendsofphp/php-cs-fixer": "^3.75.0", "illuminate/view": "^11.44.2", - "larastan/larastan": "^3.2.0", + "larastan/larastan": "^3.3.1", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -5103,7 +5147,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-03-14T22:31:42+00:00" + "time": "2025-04-08T22:11:45+00:00" }, { "name": "matthiasmullie/minify", @@ -5614,6 +5658,65 @@ ], "time": "2025-03-12T08:01:40+00:00" }, + { + "name": "phpstan/phpstan", + "version": "1.8.11", + "source": { + "type": "git", + "url": "https://github.com/phpstan/phpstan.git", + "reference": "46e223dd68a620da18855c23046ddb00940b4014" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/46e223dd68a620da18855c23046ddb00940b4014", + "reference": "46e223dd68a620da18855c23046ddb00940b4014", + "shasum": "" + }, + "require": { + "php": "^7.2|^8.0" + }, + "conflict": { + "phpstan/phpstan-shim": "*" + }, + "bin": [ + "phpstan", + "phpstan.phar" + ], + "type": "library", + "autoload": { + "files": [ + "bootstrap.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHPStan - PHP Static Analysis Tool", + "keywords": [ + "dev", + "static analysis" + ], + "support": { + "issues": "https://github.com/phpstan/phpstan/issues", + "source": "https://github.com/phpstan/phpstan/tree/1.8.11" + }, + "funding": [ + { + "url": "https://github.com/ondrejmirtes", + "type": "github" + }, + { + "url": "https://github.com/phpstan", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpstan/phpstan", + "type": "tidelift" + } + ], + "time": "2022-10-24T15:45:13+00:00" + }, { "name": "phpunit/php-code-coverage", "version": "9.2.32", @@ -8126,7 +8229,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8150,5 +8253,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.3.0" }