diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 4a1e5de227..afb27b2c94 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -4,6 +4,7 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Event; use Appwrite\Event\Migration; use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; @@ -27,8 +28,11 @@ use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; +use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Integer; use Utopia\Validator\Text; @@ -345,18 +349,50 @@ App::post('/v1/migrations/csv') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - if (!empty($file->getAttribute('openSSLCipher')) || $file->getAttribute('algorithm', Compression::NONE) !== Compression::NONE) { - throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only uncompressed, unencrypted CSV files can be used for document import."); - } + // no encryption, compression on files above 20MB. + $hasEncryption = !empty($file->getAttribute('openSSLCipher')); + $compression = $file->getAttribute('algorithm', Compression::NONE); + $hasCompression = $compression !== Compression::NONE; - // copy to temporary folder $migrationId = ID::unique(); $newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); - if (!$deviceForFiles->transfer($path, $newPath, $deviceForImports)) { + + if ($hasEncryption || $hasCompression) { + $source = $deviceForFiles->read($path); + + // 1. decrypt + if ($hasEncryption) { + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + hex2bin($file->getAttribute('openSSLIV')), + hex2bin($file->getAttribute('openSSLTag')) + ); + } + + // 2. decompress + if ($hasCompression) { + switch ($compression) { + case Compression::ZSTD: + $source = (new Zstd())->decompress($source); + break; + case Compression::GZIP: + $source = (new GZIP())->decompress($source); + break; + } + } + + // manual write after decryption and/or decompression + if (! $deviceForImports->write($newPath, $source, 'text/csv')) { + throw new \Exception("Unable to copy file"); + } + } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForImports)) { throw new \Exception("Unable to copy file"); } - $fileSize = $deviceForImports->getFileSize($path); + $fileSize = $deviceForImports->getFileSize($newPath); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); $migration = $dbForProject->createDocument('migrations', new Document([ diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 45b57d6b0c..c241b38e3d 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -971,7 +971,6 @@ trait MigrationsBase $this->assertEquals($response['body']['required'], true); // make a bucket, upload a file to it! - // 1. enable compression, encryption $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -989,33 +988,11 @@ trait MigrationsBase $bucketOneId = $bucketOne['body']['$id']; - // 2. no compression and encryption - $bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'bucketId' => ID::unique(), - 'name' => 'Test Bucket 2', - 'maximumFileSize' => 2000000, //2MB - 'allowedFileExtensions' => ['csv'], - 'compression' => 'none', - 'encryption' => false - ]); - - $this->assertNotEmpty($bucketTwo['body']['$id']); - $this->assertEquals(201, $bucketTwo['headers']['status-code']); - - $bucketTwoId = $bucketTwo['body']['$id']; - $bucketIds = [ - 'compressed' => $bucketOneId, - 'uncompressed' => $bucketTwoId, - - // in uncompressed buckets! - 'missing-row' => $bucketTwoId, - 'missing-column' => $bucketTwoId, - 'irrelevant-column' => $bucketTwoId, + 'default' => $bucketOneId, + 'missing-row' => $bucketOneId, + 'missing-column' => $bucketOneId, + 'irrelevant-column' => $bucketOneId, ]; $fileIds = []; @@ -1049,20 +1026,6 @@ trait MigrationsBase $fileIds[$label] = $response['body']['$id']; } - // compressed, fail. - $compressed = $this->performCsvMigration( - [ - 'fileId' => $fileIds['compressed'], - 'bucketId' => $bucketIds['compressed'], - 'resourceId' => $databaseId . ':' . $collectionId, - ] - ); - - // fail on compressed, encrypted buckets! - $this->assertEquals(400, $compressed['body']['code']); - $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); - $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); - // missing attribute, fail in worker. $missingColumn = $this->performCsvMigration( [ @@ -1150,12 +1113,12 @@ trait MigrationsBase ); }, 60000, 500); - // no compression, no encryption, pass. + // all data exists, pass/ $migration = $this->performCsvMigration( [ 'endpoint' => 'http://localhost/v1', - 'fileId' => $fileIds['uncompressed'], - 'bucketId' => $bucketIds['uncompressed'], + 'fileId' => $fileIds['default'], + 'bucketId' => $bucketIds['default'], 'resourceId' => $databaseId . ':' . $collectionId, ] );