diff --git a/Dockerfile b/Dockerfile index 31c05d0363..17f66b6074 100755 --- a/Dockerfile +++ b/Dockerfile @@ -44,12 +44,14 @@ COPY ./dev /usr/src/code/dev # Set Volumes RUN mkdir -p /storage/uploads && \ + mkdir -p /storage/imports && \ mkdir -p /storage/cache && \ mkdir -p /storage/config && \ mkdir -p /storage/certificates && \ mkdir -p /storage/functions && \ mkdir -p /storage/debug && \ chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ + chown -Rf www-data.www-data /storage/imports && chmod -Rf 0755 /storage/imports && \ chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 806b6cf088..ae27e81fcc 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1202,7 +1202,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('adapter'), // ssr or static + '$id' => ID::custom('adapter'), // ssr or static; named this way as it's a term in SSR frameworks 'type' => Database::VAR_STRING, 'format' => '', 'size' => 128, @@ -1727,7 +1727,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('adapter'), + '$id' => ID::custom('adapter'), // ssr or static; named this way as it's a term in SSR frameworks 'type' => Database::VAR_STRING, 'format' => '', 'size' => 128, @@ -2275,6 +2275,17 @@ return [ 'array' => false, 'filters' => ['json', 'encrypt'], ], + [ + '$id' => ID::custom('options'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65536, + 'signed' => true, + 'required' => false, + 'default' => [], + 'array' => false, + 'filters' => ['json'], + ], [ '$id' => ID::custom('resources'), 'type' => Database::VAR_STRING, @@ -2329,7 +2340,29 @@ return [ 'default' => null, 'array' => false, 'filters' => [], - ] + ], + [ + '$id' => ID::custom('resourceId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('resourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -2353,6 +2386,13 @@ return [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_resource_id', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_DESC], + ], [ '$id' => ID::custom('_fulltext_search'), 'type' => Database::INDEX_FULLTEXT, diff --git a/app/config/template-runtimes.php b/app/config/template-runtimes.php index 4a2436b2b8..8f1c0198c2 100644 --- a/app/config/template-runtimes.php +++ b/app/config/template-runtimes.php @@ -1,5 +1,8 @@ [ 'name' => 'node', diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 75afc7ed2c..4a1e5de227 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -1,5 +1,6 @@ dynamic($migration, Response::MODEL_MIGRATION); }); - App::post('/v1/migrations/firebase') ->groups(['api', 'migrations']) ->desc('Migrate Firebase data') @@ -290,6 +297,98 @@ App::post('/v1/migrations/nhost') ->dynamic($migration, Response::MODEL_MIGRATION); }); +App::post('/v1/migrations/csv') + ->groups(['api', 'migrations']) + ->desc('Import documents from a CSV') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + name: 'createCsvMigration', + description: '/docs/references/migrations/migration-csv.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('deviceForFiles') + ->inject('deviceForImports') + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForImports, Event $queueForEvents, Migration $queueForMigrations) { + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + if (!empty($file->getAttribute('openSSLCipher')) || $file->getAttribute('algorithm', Compression::NONE) !== Compression::NONE) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only uncompressed, unencrypted CSV files can be used for document import."); + } + + // copy to temporary folder + $migrationId = ID::unique(); + $newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); + if (!$deviceForFiles->transfer($path, $newPath, $deviceForImports)) { + throw new \Exception("Unable to copy file"); + } + + $fileSize = $deviceForImports->getFileSize($path); + $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => $migrationId, + 'status' => 'pending', + 'stage' => 'init', + 'source' => CSV::getName(), + 'destination' => Appwrite::getName(), + 'resources' => $resources, + 'resourceId' => $resourceId, + 'resourceType' => Resource::TYPE_DATABASE, + 'statusCounters' => [], + 'resourceData' => [], + 'errors' => [], + 'options' => [ + 'path' => $newPath, + 'size' => $fileSize, + ], + ])); + + $queueForEvents->setParam('migrationId', $migration->getId()); + + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + App::get('/v1/migrations') ->groups(['api', 'migrations']) ->desc('List migrations') diff --git a/app/init/constants.php b/app/init/constants.php index 4d57c63ef5..2b15f9fa0b 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -50,6 +50,7 @@ const APP_STORAGE_SITES = '/storage/sites'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; +const APP_STORAGE_IMPORTS = '/storage/imports'; // Temporary storage for csv imports const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` diff --git a/app/init/resources.php b/app/init/resources.php index 99f90e065f..1e6154df10 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -512,6 +512,10 @@ App::setResource('deviceForSites', function ($project) { return getDevice(APP_STORAGE_SITES . '/app-' . $project->getId()); }, ['project']); +App::setResource('deviceForImports', function (Document $project) { + return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); +}, ['project']); + App::setResource('deviceForFunctions', function ($project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/worker.php b/app/worker.php index 0792847235..3dfabfed57 100644 --- a/app/worker.php +++ b/app/worker.php @@ -343,6 +343,10 @@ Server::setResource('deviceForSites', function (Document $project) { return getDevice(APP_STORAGE_SITES . '/app-' . $project->getId()); }, ['project']); +Server::setResource('deviceForImports', function (Document $project) { + return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); +}, ['project']); + Server::setResource('deviceForFunctions', function (Document $project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/composer.json b/composer.json index 48b5a48fdc..3b5812cffb 100644 --- a/composer.json +++ b/composer.json @@ -62,7 +62,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.8.*", + "utopia-php/migration": "0.9.1", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", diff --git a/composer.lock b/composer.lock index f151523ffc..fea7bf6d51 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cac7679b9486588135dad678d9488f9e", + "content-hash": "e7875026636ccec909f9aa4d79091d5b", "packages": [ { "name": "adhocore/jwt", @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.3", + "version": "1.2.4", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc" + "reference": "47fcb66ae5328c5a799195247b1dce551d85873e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/0e7804c176c4b09d95b7985400aa38ce544cb7fc", - "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/47fcb66ae5328c5a799195247b1dce551d85873e", + "reference": "47fcb66ae5328c5a799195247b1dce551d85873e", "shasum": "" }, "require": { @@ -1451,7 +1451,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-04-08T09:55:41+00:00" + "time": "2025-04-15T07:02:07+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3351,16 +3351,16 @@ }, { "name": "utopia-php/cli", - "version": "0.15.1", + "version": "0.15.2", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65" + "reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/d69bbe51a6a94dc4e5bcdd542b5938038b985a65", - "reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/da00ff6b8b29a826a1794002ae43442cdf3a0f5f", + "reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f", "shasum": "" }, "require": { @@ -3394,9 +3394,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.15.1" + "source": "https://github.com/utopia-php/cli/tree/0.15.2" }, - "time": "2024-10-04T13:55:36+00:00" + "time": "2025-04-15T10:08:48+00:00" }, { "name": "utopia-php/compression", @@ -3996,16 +3996,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.6", + "version": "0.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "84163e16edc0b2e64c34ad7b7c4cc5f05d762daf" + "reference": "f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/84163e16edc0b2e64c34ad7b7c4cc5f05d762daf", - "reference": "84163e16edc0b2e64c34ad7b7c4cc5f05d762daf", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3", + "reference": "f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3", "shasum": "" }, "require": { @@ -4046,9 +4046,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.6" + "source": "https://github.com/utopia-php/migration/tree/0.9.1" }, - "time": "2025-04-14T08:22:09+00:00" + "time": "2025-04-17T05:18:58+00:00" }, { "name": "utopia-php/orchestration", @@ -4152,16 +4152,16 @@ }, { "name": "utopia-php/pools", - "version": "0.8.0", + "version": "0.8.2", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba" + "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/60733929dc328e7ea47e800579c8bbf0d49df5ba", - "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", + "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", "shasum": "" }, "require": { @@ -4198,9 +4198,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.8.0" + "source": "https://github.com/utopia-php/pools/tree/0.8.2" }, - "time": "2025-03-19T10:22:03+00:00" + "time": "2025-04-17T02:04:54+00:00" }, { "name": "utopia-php/preloader", @@ -4811,16 +4811,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.11", + "version": "0.40.12", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" + "reference": "182ec17848f81b78c336379bac94ff92b7a73365" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", - "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/182ec17848f81b78c336379bac94ff92b7a73365", + "reference": "182ec17848f81b78c336379bac94ff92b7a73365", "shasum": "" }, "require": { @@ -4856,9 +4856,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.12" }, - "time": "2025-03-26T10:53:16+00:00" + "time": "2025-04-02T23:36:11+00:00" }, { "name": "doctrine/annotations", diff --git a/docker-compose.yml b/docker-compose.yml index 44ab71753a..51cb511f17 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw + - appwrite-imports:/storage/imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -684,6 +685,7 @@ services: networks: - appwrite volumes: + - appwrite-imports:/storage/imports:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests @@ -1159,6 +1161,7 @@ volumes: appwrite-redis: appwrite-cache: appwrite-uploads: + appwrite-imports: appwrite-certificates: appwrite-functions: appwrite-sites: diff --git a/docs/references/migrations/migration-csv.md b/docs/references/migrations/migration-csv.md new file mode 100644 index 0000000000..7a32d5ff6e --- /dev/null +++ b/docs/references/migrations/migration-csv.md @@ -0,0 +1 @@ +Import documents from a CSV file into your Appwrite database. This endpoint allows you to import documents from a CSV file uploaded to Appwrite Storage bucket. \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 6b8543f299..b009da0718 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -18,12 +18,14 @@ use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; +use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Device; use Utopia\System\System; class Migrations extends Action @@ -32,6 +34,8 @@ class Migrations extends Action protected Database $dbForPlatform; + protected Device $deviceForImports; + protected Document $project; /** @@ -57,15 +61,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') + ->inject('deviceForImports') ->callback([$this, 'action']); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void { $payload = $message->getPayload() ?? []; + $this->deviceForImports = $deviceForImports; if (empty($payload)) { throw new Exception('Missing payload'); @@ -99,7 +105,9 @@ class Migrations extends Action protected function processSource(Document $migration): Source { $source = $migration->getAttribute('source'); + $resourceId = $migration->getAttribute('resourceId'); $credentials = $migration->getAttribute('credentials'); + $migrationOptions = $migration->getAttribute('options'); return match ($source) { Firebase::getName() => new Firebase( @@ -128,6 +136,12 @@ class Migrations extends Action $credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey'], ), + CSV::getName() => new CSV( + $resourceId, + $migrationOptions['path'], + $this->deviceForImports, + $this->dbForProject + ), default => throw new \Exception('Invalid source type'), }; } diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 6c468ee730..45b57d6b0c 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -10,8 +10,10 @@ use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Query; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; +use Utopia\Migration\Sources\CSV; trait MigrationsBase { @@ -896,4 +898,332 @@ trait MigrationsBase 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } + + /** + * Import documents from a CSV file. + */ + public function testCreateCsvMigration(): array + { + // make a database + $response = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('Test Database', $response['body']['name']); + + $databaseId = $response['body']['$id']; + + // make a collection + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'Test collection', + 'collectionId' => ID::unique(), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals($response['body']['name'], 'Test collection'); + + $collectionId = $response['body']['$id']; + + // make attributes + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'name'); + $this->assertEquals($response['body']['type'], 'string'); + $this->assertEquals($response['body']['size'], 256); + $this->assertEquals($response['body']['required'], true); + + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'age', + 'min' => 18, + 'max' => 65, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'age'); + $this->assertEquals($response['body']['type'], 'integer'); + $this->assertEquals($response['body']['min'], 18); + $this->assertEquals($response['body']['max'], 65); + $this->assertEquals($response['body']['required'], true); + + // make a bucket, upload a file to it! + // 1. enable compression, encryption + $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'gzip', + 'encryption' => true + ]); + $this->assertEquals(201, $bucketOne['headers']['status-code']); + $this->assertNotEmpty($bucketOne['body']['$id']); + + $bucketOneId = $bucketOne['body']['$id']; + + // 2. no compression and encryption + $bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket 2', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'none', + 'encryption' => false + ]); + + $this->assertNotEmpty($bucketTwo['body']['$id']); + $this->assertEquals(201, $bucketTwo['headers']['status-code']); + + $bucketTwoId = $bucketTwo['body']['$id']; + + $bucketIds = [ + 'compressed' => $bucketOneId, + 'uncompressed' => $bucketTwoId, + + // in uncompressed buckets! + 'missing-row' => $bucketTwoId, + 'missing-column' => $bucketTwoId, + 'irrelevant-column' => $bucketTwoId, + ]; + + $fileIds = []; + + foreach ($bucketIds as $label => $bucketId) { + $csvFileName = match ($label) { + 'missing-row', + 'missing-column', + 'irrelevant-column' => "{$label}.csv", + default => 'documents.csv', + }; + + $mimeType = match ($csvFileName) { + default => 'text/csv', + 'missing-row.csv' => 'text/plain', // invalid csv structure, falls back to plain text! + }; + + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/'.$csvFileName), $mimeType, $csvFileName), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals($csvFileName, $response['body']['name']); + $this->assertEquals($mimeType, $response['body']['mimeType']); + + $fileIds[$label] = $response['body']['$id']; + } + + // compressed, fail. + $compressed = $this->performCsvMigration( + [ + 'fileId' => $fileIds['compressed'], + 'bucketId' => $bucketIds['compressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + // fail on compressed, encrypted buckets! + $this->assertEquals(400, $compressed['body']['code']); + $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); + $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); + + // missing attribute, fail in worker. + $missingColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['missing-column'], + 'bucketId' => $bucketIds['missing-column'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($missingColumn, $databaseId, $collectionId) { + $migrationId = $missingColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains("CSV header mismatch. Missing attribute: 'age'") + ); + }, 60000, 500); + + // missing row data, fail in worker. + $missingColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['missing-row'], + 'bucketId' => $bucketIds['missing-row'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($missingColumn, $databaseId, $collectionId) { + $migrationId = $missingColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains('CSV row does not match the number of header columns') + ); + }, 60000, 500); + + // irrelevant column - email, fail in worker. + $irrelevantColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['irrelevant-column'], + 'bucketId' => $bucketIds['irrelevant-column'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($irrelevantColumn, $databaseId, $collectionId) { + $migrationId = $irrelevantColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains("CSV header mismatch. Unexpected attribute: 'email'") + ); + }, 60000, 500); + + // no compression, no encryption, pass. + $migration = $this->performCsvMigration( + [ + 'endpoint' => 'http://localhost/v1', + 'fileId' => $fileIds['uncompressed'], + 'bucketId' => $bucketIds['uncompressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('pending', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'migrationId' => $migration['body']['$id'], + ]; + } + + /** + * @depends testCreateCsvMigration + */ + public function testImportSuccessful(array $response): void + { + $databaseId = $response['databaseId']; + $collectionId = $response['collectionId']; + $migrationId = $response['migrationId']; + + $documentsCountInCSV = 100; + + // get migration stats + $this->assertEventually(function () use ($migrationId, $databaseId, $collectionId, $documentsCountInCSV) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $migration['body']['statusCounters']); + $this->assertEquals($documentsCountInCSV, $migration['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + }, 60000, 500); + + // get documents count + $documents = $this->client->call(Client::METHOD_GET, '/databases/'.$databaseId.'/collections/'.$collectionId.'/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + // there should be only 100! + Query::limit(150)->toString() + ] + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertIsArray($documents['body']['documents']); + $this->assertIsNumeric($documents['body']['total']); + $this->assertEquals($documentsCountInCSV, $documents['body']['total']); + } + + private function performCsvMigration(array $body): array + { + return $this->client->call(Client::METHOD_POST, '/migrations/csv', [ + 'content-type' => 'application/json', + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-appwrite-project' => $this->getProject()['$id'], + ], $body); + } } diff --git a/tests/resources/csv/documents.csv b/tests/resources/csv/documents.csv new file mode 100644 index 0000000000..ea1e33b5bd --- /dev/null +++ b/tests/resources/csv/documents.csv @@ -0,0 +1,101 @@ +$id,name,age +hxfcwpcas5xokpwe,Diamond Mendez,56 +gw8nxwf6esn3tfwf,Michael Huff,20 +xb6bxg56lral1qy9,Alyssa Rodriguez,37 +imerjq5j36y3agh2,Barbara Smith,26 +07yq9qdlhmbzmr35,Evelyn Edwards,54 +ksqo631sbhwj5ltg,Tina Richardson,41 +j7zlndgu0gbshp15,Joel Hernandez,49 +mfntvnljrcmf7h6v,Zachary Cooper,59 +5f9b01nziqu2h8ed,Brittany Spears,20 +4vxzbnzraqznk5u8,Holly White,47 +d4ywy3mtphaatbpf,Kimberly Barnes,27 +88odnk6nthyyvbal,Stephen Miller,53 +08oekee3fn7mzaa5,Yvonne Newman,41 +quw55kn9895i5e4v,Carol Kane,38 +nge6bm8ykripei6f,Doris Foster,44 +4k16i33s0xl2ypx9,Joseph Stokes,28 +q0j5rxbgid66snyf,Steve Williams,31 +n1oxun7mqq3p103y,James Carey,29 +0dbvs840jkf8i0ye,Kathryn Henry,38 +5sfaidgs1h87v15v,Christopher Landry,23 +vg3punvfu5khmf41,Jennifer Mcgee,62 +f933qydr9u5b2r11,Cathy Church,35 +wjv87y1inf8yk32s,Jose Lopez,41 +uljysdvdlcyrbrwk,William Rose,30 +ot8xtzh77j55wq0s,Sarah Ford,26 +9t76vnsv2u36s43t,Alisha Jones,61 +66y4tnty62hw8c02,Kristin Kelly,61 +2punfblazi5v16ar,Brendan Stout,40 +sxhr4nf5w2gx4wbg,Kelly Cruz,18 +68dvrqfwqnkq5el9,Samantha Martin,50 +20192l6dbeinhkh0,David Santos,46 +si0l4dgay09ebfmf,Elizabeth Carroll,22 +lhse40vbldqb6ap1,Corey Owens,46 +h5t3pslykyx3kxfm,Shelby Mueller,65 +ldc0luydrw6jub0f,Dr. Sylvia Myers,29 +voc9628xg4dsgw2y,Scott Freeman,48 +o4y0gk3gqv1ax2fz,Christopher Atkinson,21 +u1n3x4e4u7e0vzj6,Sean Diaz,31 +s36eskwtm0w7lwr7,Bobby Dyer,57 +4hjnag1p5iwvtixd,Daniel Hall,62 +m91d80oxsa216zbh,Jennifer Ramirez,65 +5hj6858zo2g85n6v,Angela Jackson,57 +8m8oihv9a1e7nn92,Kelly Lewis,36 +7azy39la0no0mxi7,Jessica Munoz,55 +47pmjkhnnqhyit8c,Kelly George,65 +6j6cpy4kgneg1mmh,Anthony Johnson,65 +tnlmtvap1zz89km9,Regina Fields,61 +6cyuvnwwqdmrpfzh,Sharon Schaefer,30 +p1v4pyu2pqodc0ey,Jacob French,62 +6npynnhjt2jd05xo,Jessica Costa,23 +wcxedf13n2e9qi4l,George Hardy,53 +yf2xlcmszk2tqeig,Andrea Allison,20 +3bf2zzv7poststwa,Kevin Ferguson,32 +c2iataz0hhv39q63,Joseph Johnson,58 +3e8npxhov4a39pvq,Ashley Martinez,18 +t7dp41tysipytywq,Charles Nixon,23 +z8cztq7c47phyfhk,Carol Dudley,40 +2636f9d8r4ipm3h6,David Weber,51 +eh3f6wxtvkjq6ykq,Scott Robinson,32 +raskbwpsje69a59h,Anthony Hardy,38 +90hn1p0b4cs9e2og,Mackenzie Owens,52 +am3swwfbo076x0v1,Brian Foster,27 +5uw7utb9lq5cfncw,Hannah Forbes,56 +cs6mbfzkzifefx6r,Lauren Reed,26 +ftw3uvztziiz9x00,Morgan Smith,28 +uhrqseeo43mozpaq,Samantha Alexander,65 +pvvmzyfc1lxor11e,Tiffany Roberts,20 +jia7bdag4abz123s,Emily Hayes,34 +h6oozcngbz8o5x4y,Rebecca Villegas,52 +9v6z1pn2f9twcy12,Donald Shah,61 +wzz3jduioso77o7f,Denise Cain,59 +u51plhgvjodkswnr,Kristine Ramirez,53 +t1uhkmiytfyc13vc,Stacey Adkins,61 +iqaqnf0ybg2ct507,Daniel Hunt,20 +idwrwv2uu4hcpv2i,Roberta Johnson,48 +2yd2hd6auetjacyo,Jason Williamson,39 +egrmdbibnjhi914x,Sandra Robinson,50 +15m1pz2bb0ercgyk,Steve Rice,25 +0i21bhkxdagjurb7,Kimberly Fritz,53 +726ofi7h5snreq67,Brianna Reynolds,33 +csqxse3wym56eim6,Alexander Williams,50 +qeaoylnrsf8p3byg,Andrew Thomas,25 +edsswobumzyzbvhf,Austin Williams,57 +hdzhzpt0ahy5hkib,Nicholas Williams,24 +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48 +3z3o73x7adyuo6w0,Stacey Smith,39 +sse2u5zlgoqrgmcf,Laura Beck,20 +rvovijmvch58r4yx,Molly Clark,51 +doe06nrx8sg5mcuv,Carmen Morris,41 +jbjdwuvj5s4kw04y,Amanda Munoz,20 +6k2ewkla7js0yw23,Rachel Collins,44 +fcxuyr4kkhrnigu1,John Alexander,18 +d25fuwlos5mk07o0,Stacy Hunter,22 +1vdai2rxmwd57oet,Eric Massey,40 +pq4jnt9izu1wlrzd,Scott Garcia,20 +lz9kfc0lty5xcz14,Cassandra Nelson,35 +pu7w6tyab5jd4we9,Aaron Johnson,50 +8dupswd2kqwdyn8v,Shannon Sherman,45 +ye466l71jthiz2p6,April Garcia,60 +xogsmfwb73l16qdt,Evan Lynn,20 diff --git a/tests/resources/csv/irrelevant-column.csv b/tests/resources/csv/irrelevant-column.csv new file mode 100644 index 0000000000..92105ceaa2 --- /dev/null +++ b/tests/resources/csv/irrelevant-column.csv @@ -0,0 +1,101 @@ +$id,name,age,email +hxfcwpcas5xokpwe,Diamond Mendez,56,diamond.mendez@example.com +gw8nxwf6esn3tfwf,Michael Huff,20,michael.huff@example.com +xb6bxg56lral1qy9,Alyssa Rodriguez,37,alyssa.rodriguez@example.com +imerjq5j36y3agh2,Barbara Smith,26,barbara.smith@example.com +07yq9qdlhmbzmr35,Evelyn Edwards,54,evelyn.edwards@example.com +ksqo631sbhwj5ltg,Tina Richardson,41,tina.richardson@example.com +j7zlndgu0gbshp15,Joel Hernandez,49,joel.hernandez@example.com +mfntvnljrcmf7h6v,Zachary Cooper,59,zachary.cooper@example.com +5f9b01nziqu2h8ed,Brittany Spears,20,brittany.spears@example.com +4vxzbnzraqznk5u8,Holly White,47,holly.white@example.com +d4ywy3mtphaatbpf,Kimberly Barnes,27,kimberly.barnes@example.com +88odnk6nthyyvbal,Stephen Miller,53,stephen.miller@example.com +08oekee3fn7mzaa5,Yvonne Newman,41,yvonne.newman@example.com +quw55kn9895i5e4v,Carol Kane,38,carol.kane@example.com +nge6bm8ykripei6f,Doris Foster,44,doris.foster@example.com +4k16i33s0xl2ypx9,Joseph Stokes,28,joseph.stokes@example.com +q0j5rxbgid66snyf,Steve Williams,31,steve.williams@example.com +n1oxun7mqq3p103y,James Carey,29,james.carey@example.com +0dbvs840jkf8i0ye,Kathryn Henry,38,kathryn.henry@example.com +5sfaidgs1h87v15v,Christopher Landry,23,christopher.landry@example.com +vg3punvfu5khmf41,Jennifer Mcgee,62,jennifer.mcgee@example.com +f933qydr9u5b2r11,Cathy Church,35,cathy.church@example.com +wjv87y1inf8yk32s,Jose Lopez,41,jose.lopez@example.com +uljysdvdlcyrbrwk,William Rose,30,william.rose@example.com +ot8xtzh77j55wq0s,Sarah Ford,26,sarah.ford@example.com +9t76vnsv2u36s43t,Alisha Jones,61,alisha.jones@example.com +66y4tnty62hw8c02,Kristin Kelly,61,kristin.kelly@example.com +2punfblazi5v16ar,Brendan Stout,40,brendan.stout@example.com +sxhr4nf5w2gx4wbg,Kelly Cruz,18,kelly.cruz@example.com +68dvrqfwqnkq5el9,Samantha Martin,50,samantha.martin@example.com +20192l6dbeinhkh0,David Santos,46,david.santos@example.com +si0l4dgay09ebfmf,Elizabeth Carroll,22,elizabeth.carroll@example.com +lhse40vbldqb6ap1,Corey Owens,46,corey.owens@example.com +h5t3pslykyx3kxfm,Shelby Mueller,65,shelby.mueller@example.com +ldc0luydrw6jub0f,Dr. Sylvia Myers,29,sylvia.myers@example.com +voc9628xg4dsgw2y,Scott Freeman,48,scott.freeman@example.com +o4y0gk3gqv1ax2fz,Christopher Atkinson,21,christopher.atkinson@example.com +u1n3x4e4u7e0vzj6,Sean Diaz,31,sean.diaz@example.com +s36eskwtm0w7lwr7,Bobby Dyer,57,bobby.dyer@example.com +4hjnag1p5iwvtixd,Daniel Hall,62,daniel.hall@example.com +m91d80oxsa216zbh,Jennifer Ramirez,65,jennifer.ramirez@example.com +5hj6858zo2g85n6v,Angela Jackson,57,angela.jackson@example.com +8m8oihv9a1e7nn92,Kelly Lewis,36,kelly.lewis@example.com +7azy39la0no0mxi7,Jessica Munoz,55,jessica.munoz@example.com +47pmjkhnnqhyit8c,Kelly George,65,kelly.george@example.com +6j6cpy4kgneg1mmh,Anthony Johnson,65,anthony.johnson@example.com +tnlmtvap1zz89km9,Regina Fields,61,regina.fields@example.com +6cyuvnwwqdmrpfzh,Sharon Schaefer,30,sharon.schaefer@example.com +p1v4pyu2pqodc0ey,Jacob French,62,jacob.french@example.com +6npynnhjt2jd05xo,Jessica Costa,23,jessica.costa@example.com +wcxedf13n2e9qi4l,George Hardy,53,george.hardy@example.com +yf2xlcmszk2tqeig,Andrea Allison,20,andrea.allison@example.com +3bf2zzv7poststwa,Kevin Ferguson,32,kevin.ferguson@example.com +c2iataz0hhv39q63,Joseph Johnson,58,joseph.johnson@example.com +3e8npxhov4a39pvq,Ashley Martinez,18,ashley.martinez@example.com +t7dp41tysipytywq,Charles Nixon,23,charles.nixon@example.com +z8cztq7c47phyfhk,Carol Dudley,40,carol.dudley@example.com +2636f9d8r4ipm3h6,David Weber,51,david.weber@example.com +eh3f6wxtvkjq6ykq,Scott Robinson,32,scott.robinson@example.com +raskbwpsje69a59h,Anthony Hardy,38,anthony.hardy@example.com +90hn1p0b4cs9e2og,Mackenzie Owens,52,mackenzie.owens@example.com +am3swwfbo076x0v1,Brian Foster,27,brian.foster@example.com +5uw7utb9lq5cfncw,Hannah Forbes,56,hannah.forbes@example.com +cs6mbfzkzifefx6r,Lauren Reed,26,lauren.reed@example.com +ftw3uvztziiz9x00,Morgan Smith,28,morgan.smith@example.com +uhrqseeo43mozpaq,Samantha Alexander,65,samantha.alexander@example.com +pvvmzyfc1lxor11e,Tiffany Roberts,20,tiffany.roberts@example.com +jia7bdag4abz123s,Emily Hayes,34,emily.hayes@example.com +h6oozcngbz8o5x4y,Rebecca Villegas,52,rebecca.villegas@example.com +9v6z1pn2f9twcy12,Donald Shah,61,donald.shah@example.com +wzz3jduioso77o7f,Denise Cain,59,denise.cain@example.com +u51plhgvjodkswnr,Kristine Ramirez,53,kristine.ramirez@example.com +t1uhkmiytfyc13vc,Stacey Adkins,61,stacey.adkins@example.com +iqaqnf0ybg2ct507,Daniel Hunt,20,daniel.hunt@example.com +idwrwv2uu4hcpv2i,Roberta Johnson,48,roberta.johnson@example.com +2yd2hd6auetjacyo,Jason Williamson,39,jason.williamson@example.com +egrmdbibnjhi914x,Sandra Robinson,50,sandra.robinson@example.com +15m1pz2bb0ercgyk,Steve Rice,25,steve.rice@example.com +0i21bhkxdagjurb7,Kimberly Fritz,53,kimberly.fritz@example.com +726ofi7h5snreq67,Brianna Reynolds,33,brianna.reynolds@example.com +csqxse3wym56eim6,Alexander Williams,50,alexander.williams@example.com +qeaoylnrsf8p3byg,Andrew Thomas,25,andrew.thomas@example.com +edsswobumzyzbvhf,Austin Williams,57,austin.williams@example.com +hdzhzpt0ahy5hkib,Nicholas Williams,24,nicholas.williams@example.com +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48,michelle.cisneros@example.com +3z3o73x7adyuo6w0,Stacey Smith,39,stacey.smith@example.com +sse2u5zlgoqrgmcf,Laura Beck,20,laura.beck@example.com +rvovijmvch58r4yx,Molly Clark,51,molly.clark@example.com +doe06nrx8sg5mcuv,Carmen Morris,41,carmen.morris@example.com +jbjdwuvj5s4kw04y,Amanda Munoz,20,amanda.munoz@example.com +6k2ewkla7js0yw23,Rachel Collins,44,rachel.collins@example.com +fcxuyr4kkhrnigu1,John Alexander,18,john.alexander@example.com +d25fuwlos5mk07o0,Stacy Hunter,22,stacy.hunter@example.com +1vdai2rxmwd57oet,Eric Massey,40,eric.massey@example.com +pq4jnt9izu1wlrzd,Scott Garcia,20,scott.garcia@example.com +lz9kfc0lty5xcz14,Cassandra Nelson,35,cassandra.nelson@example.com +pu7w6tyab5jd4we9,Aaron Johnson,50,aaron.johnson@example.com +8dupswd2kqwdyn8v,Shannon Sherman,45,shannon.sherman@example.com +ye466l71jthiz2p6,April Garcia,60,april.garcia@example.com +xogsmfwb73l16qdt,Evan Lynn,20,evan.lynn@example.com diff --git a/tests/resources/csv/missing-column.csv b/tests/resources/csv/missing-column.csv new file mode 100644 index 0000000000..e57b5ccb2e --- /dev/null +++ b/tests/resources/csv/missing-column.csv @@ -0,0 +1,101 @@ +$id,name +hxfcwpcas5xokpwe,Diamond Mendez +gw8nxwf6esn3tfwf,Michael Huff +xb6bxg56lral1qy9,Alyssa Rodriguez +imerjq5j36y3agh2,Barbara Smith +07yq9qdlhmbzmr35,Evelyn Edwards +ksqo631sbhwj5ltg,Tina Richardson +j7zlndgu0gbshp15,Joel Hernandez +mfntvnljrcmf7h6v,Zachary Cooper +5f9b01nziqu2h8ed,Brittany Spears +4vxzbnzraqznk5u8,Holly White +d4ywy3mtphaatbpf,Kimberly Barnes +88odnk6nthyyvbal,Stephen Miller +08oekee3fn7mzaa5,Yvonne Newman +quw55kn9895i5e4v,Carol Kane +nge6bm8ykripei6f,Doris Foster +4k16i33s0xl2ypx9,Joseph Stokes +q0j5rxbgid66snyf,Steve Williams +n1oxun7mqq3p103y,James Carey +0dbvs840jkf8i0ye,Kathryn Henry +5sfaidgs1h87v15v,Christopher Landry +vg3punvfu5khmf41,Jennifer Mcgee +f933qydr9u5b2r11,Cathy Church +wjv87y1inf8yk32s,Jose Lopez +uljysdvdlcyrbrwk,William Rose +ot8xtzh77j55wq0s,Sarah Ford +9t76vnsv2u36s43t,Alisha Jones +66y4tnty62hw8c02,Kristin Kelly +2punfblazi5v16ar,Brendan Stout +sxhr4nf5w2gx4wbg,Kelly Cruz +68dvrqfwqnkq5el9,Samantha Martin +20192l6dbeinhkh0,David Santos +si0l4dgay09ebfmf,Elizabeth Carroll +lhse40vbldqb6ap1,Corey Owens +h5t3pslykyx3kxfm,Shelby Mueller +ldc0luydrw6jub0f,Dr. Sylvia Myers +voc9628xg4dsgw2y,Scott Freeman +o4y0gk3gqv1ax2fz,Christopher Atkinson +u1n3x4e4u7e0vzj6,Sean Diaz +s36eskwtm0w7lwr7,Bobby Dyer +4hjnag1p5iwvtixd,Daniel Hall +m91d80oxsa216zbh,Jennifer Ramirez +5hj6858zo2g85n6v,Angela Jackson +8m8oihv9a1e7nn92,Kelly Lewis +7azy39la0no0mxi7,Jessica Munoz +47pmjkhnnqhyit8c,Kelly George +6j6cpy4kgneg1mmh,Anthony Johnson +tnlmtvap1zz89km9,Regina Fields +6cyuvnwwqdmrpfzh,Sharon Schaefer +p1v4pyu2pqodc0ey,Jacob French +6npynnhjt2jd05xo,Jessica Costa +wcxedf13n2e9qi4l,George Hardy +yf2xlcmszk2tqeig,Andrea Allison +3bf2zzv7poststwa,Kevin Ferguson +c2iataz0hhv39q63,Joseph Johnson +3e8npxhov4a39pvq,Ashley Martinez +t7dp41tysipytywq,Charles Nixon +z8cztq7c47phyfhk,Carol Dudley +2636f9d8r4ipm3h6,David Weber +eh3f6wxtvkjq6ykq,Scott Robinson +raskbwpsje69a59h,Anthony Hardy +90hn1p0b4cs9e2og,Mackenzie Owens +am3swwfbo076x0v1,Brian Foster +5uw7utb9lq5cfncw,Hannah Forbes +cs6mbfzkzifefx6r,Lauren Reed +ftw3uvztziiz9x00,Morgan Smith +uhrqseeo43mozpaq,Samantha Alexander +pvvmzyfc1lxor11e,Tiffany Roberts +jia7bdag4abz123s,Emily Hayes +h6oozcngbz8o5x4y,Rebecca Villegas +9v6z1pn2f9twcy12,Donald Shah +wzz3jduioso77o7f,Denise Cain +u51plhgvjodkswnr,Kristine Ramirez +t1uhkmiytfyc13vc,Stacey Adkins +iqaqnf0ybg2ct507,Daniel Hunt +idwrwv2uu4hcpv2i,Roberta Johnson +2yd2hd6auetjacyo,Jason Williamson +egrmdbibnjhi914x,Sandra Robinson +15m1pz2bb0ercgyk,Steve Rice +0i21bhkxdagjurb7,Kimberly Fritz +726ofi7h5snreq67,Brianna Reynolds +csqxse3wym56eim6,Alexander Williams +qeaoylnrsf8p3byg,Andrew Thomas +edsswobumzyzbvhf,Austin Williams +hdzhzpt0ahy5hkib,Nicholas Williams +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros +3z3o73x7adyuo6w0,Stacey Smith +sse2u5zlgoqrgmcf,Laura Beck +rvovijmvch58r4yx,Molly Clark +doe06nrx8sg5mcuv,Carmen Morris +jbjdwuvj5s4kw04y,Amanda Munoz +6k2ewkla7js0yw23,Rachel Collins +fcxuyr4kkhrnigu1,John Alexander +d25fuwlos5mk07o0,Stacy Hunter +1vdai2rxmwd57oet,Eric Massey +pq4jnt9izu1wlrzd,Scott Garcia +lz9kfc0lty5xcz14,Cassandra Nelson +pu7w6tyab5jd4we9,Aaron Johnson +8dupswd2kqwdyn8v,Shannon Sherman +ye466l71jthiz2p6,April Garcia +xogsmfwb73l16qdt,Evan Lynn diff --git a/tests/resources/csv/missing-row.csv b/tests/resources/csv/missing-row.csv new file mode 100644 index 0000000000..7399fa9f51 --- /dev/null +++ b/tests/resources/csv/missing-row.csv @@ -0,0 +1,101 @@ +$id,name,age +hxfcwpcas5xokpwe,Diamond Mendez +gw8nxwf6esn3tfwf,Michael Huff +xb6bxg56lral1qy9,Alyssa Rodriguez +imerjq5j36y3agh2,Barbara Smith +07yq9qdlhmbzmr35,Evelyn Edwards +ksqo631sbhwj5ltg,Tina Richardson +j7zlndgu0gbshp15,Joel Hernandez +mfntvnljrcmf7h6v,Zachary Cooper +5f9b01nziqu2h8ed,Brittany Spears +4vxzbnzraqznk5u8,Holly White +d4ywy3mtphaatbpf,Kimberly Barnes +88odnk6nthyyvbal,Stephen Miller +08oekee3fn7mzaa5,Yvonne Newman +quw55kn9895i5e4v,Carol Kane +nge6bm8ykripei6f,Doris Foster +4k16i33s0xl2ypx9,Joseph Stokes +q0j5rxbgid66snyf,Steve Williams +n1oxun7mqq3p103y,James Carey +0dbvs840jkf8i0ye,Kathryn Henry +5sfaidgs1h87v15v,Christopher Landry +vg3punvfu5khmf41,Jennifer Mcgee +f933qydr9u5b2r11,Cathy Church +wjv87y1inf8yk32s,Jose Lopez +uljysdvdlcyrbrwk,William Rose +ot8xtzh77j55wq0s,Sarah Ford +9t76vnsv2u36s43t,Alisha Jones +66y4tnty62hw8c02,Kristin Kelly +2punfblazi5v16ar,Brendan Stout +sxhr4nf5w2gx4wbg,Kelly Cruz +68dvrqfwqnkq5el9,Samantha Martin +20192l6dbeinhkh0,David Santos +si0l4dgay09ebfmf,Elizabeth Carroll +lhse40vbldqb6ap1,Corey Owens +h5t3pslykyx3kxfm,Shelby Mueller +ldc0luydrw6jub0f,Dr. Sylvia Myers +voc9628xg4dsgw2y,Scott Freeman +o4y0gk3gqv1ax2fz,Christopher Atkinson +u1n3x4e4u7e0vzj6,Sean Diaz +s36eskwtm0w7lwr7,Bobby Dyer +4hjnag1p5iwvtixd,Daniel Hall +m91d80oxsa216zbh,Jennifer Ramirez +5hj6858zo2g85n6v,Angela Jackson +8m8oihv9a1e7nn92,Kelly Lewis +7azy39la0no0mxi7,Jessica Munoz +47pmjkhnnqhyit8c,Kelly George +6j6cpy4kgneg1mmh,Anthony Johnson +tnlmtvap1zz89km9,Regina Fields +6cyuvnwwqdmrpfzh,Sharon Schaefer +p1v4pyu2pqodc0ey,Jacob French +6npynnhjt2jd05xo,Jessica Costa +wcxedf13n2e9qi4l,George Hardy +yf2xlcmszk2tqeig,Andrea Allison +3bf2zzv7poststwa,Kevin Ferguson +c2iataz0hhv39q63,Joseph Johnson +3e8npxhov4a39pvq,Ashley Martinez +t7dp41tysipytywq,Charles Nixon +z8cztq7c47phyfhk,Carol Dudley +2636f9d8r4ipm3h6,David Weber +eh3f6wxtvkjq6ykq,Scott Robinson +raskbwpsje69a59h,Anthony Hardy +90hn1p0b4cs9e2og,Mackenzie Owens +am3swwfbo076x0v1,Brian Foster +5uw7utb9lq5cfncw,Hannah Forbes +cs6mbfzkzifefx6r,Lauren Reed +ftw3uvztziiz9x00,Morgan Smith +uhrqseeo43mozpaq,Samantha Alexander +pvvmzyfc1lxor11e,Tiffany Roberts +jia7bdag4abz123s,Emily Hayes +h6oozcngbz8o5x4y,Rebecca Villegas +9v6z1pn2f9twcy12,Donald Shah +wzz3jduioso77o7f,Denise Cain +u51plhgvjodkswnr,Kristine Ramirez +t1uhkmiytfyc13vc,Stacey Adkins +iqaqnf0ybg2ct507,Daniel Hunt +idwrwv2uu4hcpv2i,Roberta Johnson +2yd2hd6auetjacyo,Jason Williamson +egrmdbibnjhi914x,Sandra Robinson +15m1pz2bb0ercgyk,Steve Rice +0i21bhkxdagjurb7,Kimberly Fritz +726ofi7h5snreq67,Brianna Reynolds +csqxse3wym56eim6,Alexander Williams +qeaoylnrsf8p3byg,Andrew Thomas +edsswobumzyzbvhf,Austin Williams +hdzhzpt0ahy5hkib,Nicholas Williams +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros +3z3o73x7adyuo6w0,Stacey Smith +sse2u5zlgoqrgmcf,Laura Beck +rvovijmvch58r4yx,Molly Clark +doe06nrx8sg5mcuv,Carmen Morris +jbjdwuvj5s4kw04y,Amanda Munoz +6k2ewkla7js0yw23,Rachel Collins +fcxuyr4kkhrnigu1,John Alexander +d25fuwlos5mk07o0,Stacy Hunter +1vdai2rxmwd57oet,Eric Massey +pq4jnt9izu1wlrzd,Scott Garcia +lz9kfc0lty5xcz14,Cassandra Nelson +pu7w6tyab5jd4we9,Aaron Johnson +8dupswd2kqwdyn8v,Shannon Sherman +ye466l71jthiz2p6,April Garcia +xogsmfwb73l16qdt,Evan Lynn