From 873b384f70c83486da721d333a1f99f33235fc3f Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 21 Apr 2025 10:01:13 +0530 Subject: [PATCH 1/5] update: allow compressed files. --- app/controllers/api/migrations.php | 31 +++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 4a1e5de227..a19c6d3a22 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -27,6 +27,8 @@ use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; use Utopia\Validator\ArrayList; @@ -345,18 +347,37 @@ App::post('/v1/migrations/csv') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - if (!empty($file->getAttribute('openSSLCipher')) || $file->getAttribute('algorithm', Compression::NONE) !== Compression::NONE) { - throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only uncompressed, unencrypted CSV files can be used for document import."); + if (!empty($file->getAttribute('openSSLCipher'))) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only unencrypted CSV files can be used for document import."); } - // copy to temporary folder + $compression = $file->getAttribute('algorithm', Compression::NONE); + $hasCompression = $compression !== Compression::NONE; // skipped on files that are 20MB+ in size. + + // copy to import volume $migrationId = ID::unique(); $newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); - if (!$deviceForFiles->transfer($path, $newPath, $deviceForImports)) { + + if ($hasCompression) { + $source = $deviceForFiles->read($path); + + switch ($compression) { + case Compression::ZSTD: + $source = (new Zstd())->decompress($source); + break; + case Compression::GZIP: + $source = (new GZIP())->decompress($source); + break; + } + + if (! $deviceForImports->write($newPath, $source, 'text/csv')) { + throw new \Exception("Unable to copy file"); + } + } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForImports)) { throw new \Exception("Unable to copy file"); } - $fileSize = $deviceForImports->getFileSize($path); + $fileSize = $deviceForImports->getFileSize($newPath); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); $migration = $dbForProject->createDocument('migrations', new Document([ From 21be8190252521546082fee380ce3da611d2ec77 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 21 Apr 2025 10:12:09 +0530 Subject: [PATCH 2/5] fix: test message! --- tests/e2e/Services/Migrations/MigrationsBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 45b57d6b0c..2628500c7d 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1061,7 +1061,7 @@ trait MigrationsBase // fail on compressed, encrypted buckets! $this->assertEquals(400, $compressed['body']['code']); $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); - $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); + $this->assertEquals('Only unencrypted CSV files can be used for document import.', $compressed['body']['message']); // missing attribute, fail in worker. $missingColumn = $this->performCsvMigration( From 0accc494f0e1d720440b88cc34c253c9d2ed1701 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 21 Apr 2025 10:25:43 +0530 Subject: [PATCH 3/5] update: tests. --- .../Services/Migrations/MigrationsBase.php | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 2628500c7d..1c6698e53f 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -971,7 +971,7 @@ trait MigrationsBase $this->assertEquals($response['body']['required'], true); // make a bucket, upload a file to it! - // 1. enable compression, encryption + // 1. enable encryption $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -989,7 +989,7 @@ trait MigrationsBase $bucketOneId = $bucketOne['body']['$id']; - // 2. no compression and encryption + // 2. no encryption $bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -999,7 +999,7 @@ trait MigrationsBase 'name' => 'Test Bucket 2', 'maximumFileSize' => 2000000, //2MB 'allowedFileExtensions' => ['csv'], - 'compression' => 'none', + 'compression' => 'gzip', 'encryption' => false ]); @@ -1009,10 +1009,10 @@ trait MigrationsBase $bucketTwoId = $bucketTwo['body']['$id']; $bucketIds = [ - 'compressed' => $bucketOneId, - 'uncompressed' => $bucketTwoId, + 'encrypted' => $bucketOneId, + 'unencrypted' => $bucketTwoId, - // in uncompressed buckets! + // in unencrypted buckets! 'missing-row' => $bucketTwoId, 'missing-column' => $bucketTwoId, 'irrelevant-column' => $bucketTwoId, @@ -1049,19 +1049,19 @@ trait MigrationsBase $fileIds[$label] = $response['body']['$id']; } - // compressed, fail. - $compressed = $this->performCsvMigration( + // encrypted bucket, fail. + $encrypted = $this->performCsvMigration( [ - 'fileId' => $fileIds['compressed'], - 'bucketId' => $bucketIds['compressed'], + 'fileId' => $fileIds['encrypted'], + 'bucketId' => $bucketIds['encrypted'], 'resourceId' => $databaseId . ':' . $collectionId, ] ); // fail on compressed, encrypted buckets! - $this->assertEquals(400, $compressed['body']['code']); - $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); - $this->assertEquals('Only unencrypted CSV files can be used for document import.', $compressed['body']['message']); + $this->assertEquals(400, $encrypted['body']['code']); + $this->assertEquals('storage_file_type_unsupported', $encrypted['body']['type']); + $this->assertEquals('Only unencrypted CSV files can be used for document import.', $encrypted['body']['message']); // missing attribute, fail in worker. $missingColumn = $this->performCsvMigration( @@ -1150,12 +1150,12 @@ trait MigrationsBase ); }, 60000, 500); - // no compression, no encryption, pass. + // no encryption, pass. $migration = $this->performCsvMigration( [ 'endpoint' => 'http://localhost/v1', - 'fileId' => $fileIds['uncompressed'], - 'bucketId' => $bucketIds['uncompressed'], + 'fileId' => $fileIds['unencrypted'], + 'bucketId' => $bucketIds['unencrypted'], 'resourceId' => $databaseId . ':' . $collectionId, ] ); From 927489bc5b4bae5ef561d1bbb8f6ee27c68030a8 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 21 Apr 2025 12:04:36 +0530 Subject: [PATCH 4/5] update: handle decryption as well. --- app/controllers/api/migrations.php | 43 ++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index a19c6d3a22..afb27b2c94 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -4,6 +4,7 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Event; use Appwrite\Event\Migration; use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; @@ -31,6 +32,7 @@ use Utopia\Storage\Compression\Algorithms\GZIP; use Utopia\Storage\Compression\Algorithms\Zstd; use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; +use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Integer; use Utopia\Validator\Text; @@ -347,29 +349,42 @@ App::post('/v1/migrations/csv') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - if (!empty($file->getAttribute('openSSLCipher'))) { - throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only unencrypted CSV files can be used for document import."); - } - + // no encryption, compression on files above 20MB. + $hasEncryption = !empty($file->getAttribute('openSSLCipher')); $compression = $file->getAttribute('algorithm', Compression::NONE); - $hasCompression = $compression !== Compression::NONE; // skipped on files that are 20MB+ in size. + $hasCompression = $compression !== Compression::NONE; - // copy to import volume $migrationId = ID::unique(); $newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); - if ($hasCompression) { + if ($hasEncryption || $hasCompression) { $source = $deviceForFiles->read($path); - switch ($compression) { - case Compression::ZSTD: - $source = (new Zstd())->decompress($source); - break; - case Compression::GZIP: - $source = (new GZIP())->decompress($source); - break; + // 1. decrypt + if ($hasEncryption) { + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + hex2bin($file->getAttribute('openSSLIV')), + hex2bin($file->getAttribute('openSSLTag')) + ); } + // 2. decompress + if ($hasCompression) { + switch ($compression) { + case Compression::ZSTD: + $source = (new Zstd())->decompress($source); + break; + case Compression::GZIP: + $source = (new GZIP())->decompress($source); + break; + } + } + + // manual write after decryption and/or decompression if (! $deviceForImports->write($newPath, $source, 'text/csv')) { throw new \Exception("Unable to copy file"); } From 53a0424326705a8126ef4486ae91664937cc869d Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 21 Apr 2025 12:12:01 +0530 Subject: [PATCH 5/5] update: tests. --- .../Services/Migrations/MigrationsBase.php | 51 +++---------------- 1 file changed, 7 insertions(+), 44 deletions(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 1c6698e53f..c241b38e3d 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -971,7 +971,6 @@ trait MigrationsBase $this->assertEquals($response['body']['required'], true); // make a bucket, upload a file to it! - // 1. enable encryption $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -989,33 +988,11 @@ trait MigrationsBase $bucketOneId = $bucketOne['body']['$id']; - // 2. no encryption - $bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'bucketId' => ID::unique(), - 'name' => 'Test Bucket 2', - 'maximumFileSize' => 2000000, //2MB - 'allowedFileExtensions' => ['csv'], - 'compression' => 'gzip', - 'encryption' => false - ]); - - $this->assertNotEmpty($bucketTwo['body']['$id']); - $this->assertEquals(201, $bucketTwo['headers']['status-code']); - - $bucketTwoId = $bucketTwo['body']['$id']; - $bucketIds = [ - 'encrypted' => $bucketOneId, - 'unencrypted' => $bucketTwoId, - - // in unencrypted buckets! - 'missing-row' => $bucketTwoId, - 'missing-column' => $bucketTwoId, - 'irrelevant-column' => $bucketTwoId, + 'default' => $bucketOneId, + 'missing-row' => $bucketOneId, + 'missing-column' => $bucketOneId, + 'irrelevant-column' => $bucketOneId, ]; $fileIds = []; @@ -1049,20 +1026,6 @@ trait MigrationsBase $fileIds[$label] = $response['body']['$id']; } - // encrypted bucket, fail. - $encrypted = $this->performCsvMigration( - [ - 'fileId' => $fileIds['encrypted'], - 'bucketId' => $bucketIds['encrypted'], - 'resourceId' => $databaseId . ':' . $collectionId, - ] - ); - - // fail on compressed, encrypted buckets! - $this->assertEquals(400, $encrypted['body']['code']); - $this->assertEquals('storage_file_type_unsupported', $encrypted['body']['type']); - $this->assertEquals('Only unencrypted CSV files can be used for document import.', $encrypted['body']['message']); - // missing attribute, fail in worker. $missingColumn = $this->performCsvMigration( [ @@ -1150,12 +1113,12 @@ trait MigrationsBase ); }, 60000, 500); - // no encryption, pass. + // all data exists, pass/ $migration = $this->performCsvMigration( [ 'endpoint' => 'http://localhost/v1', - 'fileId' => $fileIds['unencrypted'], - 'bucketId' => $bucketIds['unencrypted'], + 'fileId' => $fileIds['default'], + 'bucketId' => $bucketIds['default'], 'resourceId' => $databaseId . ':' . $collectionId, ] );