Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-22 16:38:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch '1.7.x' of github.com:appwrite/appwrite into 1.7.x

Browse source
This commit is contained in:
Khushboo Verma 2025-04-22 12:46:33 +05:30
commit 1f51381e4b
2 changed files with 49 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
app/controllers/api/migrations.php
View file

@ -4,6 +4,7 @@ use Appwrite\Auth\Auth;
use Appwrite\Event\Event;
use Appwrite\Event\Migration;
use Appwrite\Extend\Exception;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\ContentType;
use Appwrite\SDK\Method;
@ -27,8 +28,11 @@ use Utopia\Migration\Sources\Firebase;
use Utopia\Migration\Sources\NHost;
use Utopia\Migration\Sources\Supabase;
use Utopia\Migration\Transfer;
use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Storage\Compression\Algorithms\Zstd;
use Utopia\Storage\Compression\Compression;
use Utopia\Storage\Device;
use Utopia\System\System;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Integer;
use Utopia\Validator\Text;
@ -345,18 +349,50 @@ App::post('/v1/migrations/csv')
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path);
}
if (!empty($file->getAttribute('openSSLCipher')) || $file->getAttribute('algorithm', Compression::NONE) !== Compression::NONE) {
throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only uncompressed, unencrypted CSV files can be used for document import.");
}
// no encryption, compression on files above 20MB.
$hasEncryption = !empty($file->getAttribute('openSSLCipher'));
$compression = $file->getAttribute('algorithm', Compression::NONE);
$hasCompression = $compression !== Compression::NONE;
// copy to temporary folder
$migrationId = ID::unique();
$newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv');
if (!$deviceForFiles->transfer($path, $newPath, $deviceForImports)) {
if ($hasEncryption || $hasCompression) {
$source = $deviceForFiles->read($path);
// 1. decrypt
if ($hasEncryption) {
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
hex2bin($file->getAttribute('openSSLIV')),
hex2bin($file->getAttribute('openSSLTag'))
);
}
// 2. decompress
if ($hasCompression) {
switch ($compression) {
case Compression::ZSTD:
$source = (new Zstd())->decompress($source);
break;
case Compression::GZIP:
$source = (new GZIP())->decompress($source);
break;
}
}
// manual write after decryption and/or decompression
if (! $deviceForImports->write($newPath, $source, 'text/csv')) {
throw new \Exception("Unable to copy file");
}
} elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForImports)) {
throw new \Exception("Unable to copy file");
}
$fileSize = $deviceForImports->getFileSize($path);
$fileSize = $deviceForImports->getFileSize($newPath);
$resources = Transfer::extractServices([Transfer::GROUP_DATABASES]);
$migration = $dbForProject->createDocument('migrations', new Document([

51
tests/e2e/Services/Migrations/MigrationsBase.php
View file

@ -971,7 +971,6 @@ trait MigrationsBase
$this->assertEquals($response['body']['required'], true);
// make a bucket, upload a file to it!
// 1. enable compression, encryption
$bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@ -989,33 +988,11 @@ trait MigrationsBase
$bucketOneId = $bucketOne['body']['$id'];
// 2. no compression and encryption
$bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket 2',
'maximumFileSize' => 2000000, //2MB
'allowedFileExtensions' => ['csv'],
'compression' => 'none',
'encryption' => false
]);
$this->assertNotEmpty($bucketTwo['body']['$id']);
$this->assertEquals(201, $bucketTwo['headers']['status-code']);
$bucketTwoId = $bucketTwo['body']['$id'];
$bucketIds = [
'compressed' => $bucketOneId,
'uncompressed' => $bucketTwoId,
// in uncompressed buckets!
'missing-row' => $bucketTwoId,
'missing-column' => $bucketTwoId,
'irrelevant-column' => $bucketTwoId,
'default' => $bucketOneId,
'missing-row' => $bucketOneId,
'missing-column' => $bucketOneId,
'irrelevant-column' => $bucketOneId,
];
$fileIds = [];
@ -1049,20 +1026,6 @@ trait MigrationsBase
$fileIds[$label] = $response['body']['$id'];
}
// compressed, fail.
$compressed = $this->performCsvMigration(
[
'fileId' => $fileIds['compressed'],
'bucketId' => $bucketIds['compressed'],
'resourceId' => $databaseId . ':' . $collectionId,
]
);
// fail on compressed, encrypted buckets!
$this->assertEquals(400, $compressed['body']['code']);
$this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']);
$this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']);
// missing attribute, fail in worker.
$missingColumn = $this->performCsvMigration(
[
@ -1150,12 +1113,12 @@ trait MigrationsBase
);
}, 60000, 500);
// no compression, no encryption, pass.
// all data exists, pass/
$migration = $this->performCsvMigration(
[
'endpoint' => 'http://localhost/v1',
'fileId' => $fileIds['uncompressed'],
'bucketId' => $bucketIds['uncompressed'],
'fileId' => $fileIds['default'],
'bucketId' => $bucketIds['default'],
'resourceId' => $databaseId . ':' . $collectionId,
]
);