Merge pull request #10813 from appwrite/feat-csv-export

Feat csv export
This commit is contained in:
Jake Barnby 2025-11-14 04:16:31 +00:00 committed by GitHub
commit e783f4c234
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 100 additions and 146 deletions

View file

@ -23026,7 +23026,7 @@
"tags": [
"migrations"
],
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.",
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.",
"responses": {
"202": {
"description": "Migration",
@ -23076,11 +23076,6 @@
"description": "Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.",
"x-example": "<ID1:ID2>"
},
"bucketId": {
"type": "string",
"description": "Storage bucket unique ID where the exported CSV will be stored.",
"x-example": "<BUCKET_ID>"
},
"filename": {
"type": "string",
"description": "The name of the file to be created for the export, excluding the .csv extension.",
@ -23130,7 +23125,6 @@
},
"required": [
"resourceId",
"bucketId",
"filename"
]
}

View file

@ -23039,7 +23039,7 @@
"tags": [
"migrations"
],
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.",
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.",
"responses": {
"202": {
"description": "Migration",
@ -23089,11 +23089,6 @@
"description": "Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.",
"x-example": "<ID1:ID2>"
},
"bucketId": {
"type": "string",
"description": "Storage bucket unique ID where the exported CSV will be stored.",
"x-example": "<BUCKET_ID>"
},
"filename": {
"type": "string",
"description": "The name of the file to be created for the export, excluding the .csv extension.",
@ -23143,7 +23138,6 @@
},
"required": [
"resourceId",
"bucketId",
"filename"
]
}

View file

@ -23127,7 +23127,7 @@
"tags": [
"migrations"
],
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.",
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.",
"responses": {
"202": {
"description": "Migration",
@ -23175,12 +23175,6 @@
"default": null,
"x-example": "<ID1:ID2>"
},
"bucketId": {
"type": "string",
"description": "Storage bucket unique ID where the exported CSV will be stored.",
"default": null,
"x-example": "<BUCKET_ID>"
},
"filename": {
"type": "string",
"description": "The name of the file to be created for the export, excluding the .csv extension.",
@ -23238,7 +23232,6 @@
},
"required": [
"resourceId",
"bucketId",
"filename"
]
}

View file

@ -23141,7 +23141,7 @@
"tags": [
"migrations"
],
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.",
"description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.",
"responses": {
"202": {
"description": "Migration",
@ -23189,12 +23189,6 @@
"default": null,
"x-example": "<ID1:ID2>"
},
"bucketId": {
"type": "string",
"description": "Storage bucket unique ID where the exported CSV will be stored.",
"default": null,
"x-example": "<BUCKET_ID>"
},
"filename": {
"type": "string",
"description": "The name of the file to be created for the export, excluding the .csv extension.",
@ -23252,7 +23246,6 @@
},
"required": [
"resourceId",
"bucketId",
"filename"
]
}

View file

@ -468,7 +468,6 @@ App::post('/v1/migrations/csv/exports')
]
))
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.')
->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.')
->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.')
->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true)
->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
@ -480,12 +479,12 @@ App::post('/v1/migrations/csv/exports')
->inject('user')
->inject('response')
->inject('dbForProject')
->inject('dbForPlatform')
->inject('project')
->inject('queueForEvents')
->inject('queueForMigrations')
->action(function (
string $resourceId,
string $bucketId,
string $filename,
array $columns,
array $queries,
@ -497,6 +496,7 @@ App::post('/v1/migrations/csv/exports')
Document $user,
Response $response,
Database $dbForProject,
Database $dbForPlatform,
Document $project,
Event $queueForEvents,
Migration $queueForMigrations
@ -507,7 +507,7 @@ App::post('/v1/migrations/csv/exports')
throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage());
}
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
$bucket = Authorization::skip(fn () => $dbForPlatform->getDocument('buckets', 'default'));
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@ -553,7 +553,7 @@ App::post('/v1/migrations/csv/exports')
'resourceData' => '{}',
'errors' => [],
'options' => [
'bucketId' => $bucketId,
'bucketId' => 'default', // Always use internal bucket
'filename' => $filename,
'columns' => $columns,
'queries' => $queries,

View file

@ -1478,12 +1478,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/push')
->inject('response')
->inject('request')
->inject('dbForProject')
->inject('dbForPlatform')
->inject('project')
->inject('mode')
->inject('deviceForFiles')
->action(function (string $bucketId, string $fileId, string $jwt, Response $response, Request $request, Database $dbForProject, Document $project, string $mode, Device $deviceForFiles) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
->action(function (string $bucketId, string $fileId, string $jwt, Response $response, Request $request, Database $dbForProject, Database $dbForPlatform, Document $project, string $mode, Device $deviceForFiles) {
$decoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0);
try {
@ -1500,15 +1499,18 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/push')
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$isInternal = $decoded['internal'] ?? false;
$dbForProject = $isInternal ? $dbForPlatform : $dbForProject;
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
}

View file

@ -138,6 +138,7 @@ const DELETE_TYPE_TOPIC = 'topic';
const DELETE_TYPE_TARGET = 'target';
const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets';
const DELETE_TYPE_SESSION_TARGETS = 'session_targets';
const DELETE_TYPE_CSV_EXPORTS = 'csv_exports';
const DELETE_TYPE_MAINTENANCE = 'maintenance';
// Message types

View file

@ -1,4 +1,3 @@
appwrite migrations create-csv-export \
--resource-id <ID1:ID2> \
--bucket-id <BUCKET_ID> \
--filename <FILENAME>

View file

@ -8,7 +8,6 @@ const migrations = new Migrations(client);
const result = await migrations.createCSVExport({
resourceId: '<ID1:ID2>',
bucketId: '<BUCKET_ID>',
filename: '<FILENAME>',
columns: [], // optional
queries: [], // optional

View file

@ -1 +1 @@
Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.
Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.

View file

@ -95,6 +95,7 @@ class Maintenance extends Action
$this->renewCertificates($dbForPlatform, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
$this->notifyDeleteCSVExports($queueForDeletes);
}, $interval, $delay);
}
@ -106,6 +107,13 @@ class Maintenance extends Action
->trigger();
}
private function notifyDeleteCSVExports(Delete $queueForDeletes): void
{
$queueForDeletes
->setType(DELETE_TYPE_CSV_EXPORTS)
->trigger();
}
private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void
{
$time = DatabaseDateTime::now();

View file

@ -179,6 +179,9 @@ class Deletes extends Action
case DELETE_TYPE_SESSION_TARGETS:
$this->deleteSessionTargets($project, $getProjectDB, $document);
break;
case DELETE_TYPE_CSV_EXPORTS:
$this->deleteOldCSVExports($dbForPlatform, $deviceForFiles);
break;
case DELETE_TYPE_MAINTENANCE:
$this->deleteExpiredTargets($project, $getProjectDB);
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
@ -720,6 +723,41 @@ class Deletes extends Action
], $dbForProject);
}
/**
* @param Database $dbForPlatform
* @param Device $deviceForFiles
* @return void
* @throws Exception|Throwable
*/
private function deleteOldCSVExports(Database $dbForPlatform, Device $deviceForFiles): void
{
$bucket = $dbForPlatform->getDocument('buckets', 'default');
if ($bucket->isEmpty()) {
Console::warning('Default bucket not found, skipping CSV export cleanup');
return;
}
$oneWeekAgo = DateTime::addSeconds(new \DateTime(), -1 * 60 * 60 * 24 * 7); // 1 week
Console::info("Deleting CSV export files older than " . $oneWeekAgo);
$this->deleteByGroup('bucket_' . $bucket->getSequence(), [
Query::select([...$this->selects, '$createdAt', 'name', 'path']),
Query::equal('bucketId', ['default']),
Query::createdBefore($oneWeekAgo),
Query::endsWith('name', ['.csv']),
Query::orderDesc('$createdAt'),
Query::orderDesc(),
], $dbForPlatform, function (Document $file) use ($deviceForFiles) {
$path = $file->getAttribute('path');
if ($deviceForFiles->exists($path)) {
$deviceForFiles->delete($path);
Console::success('Deleted CSV file: ' . $file->getAttribute('name'));
}
});
}
/**
* @param Database $dbForPlatform
* @param string $datetime

View file

@ -11,12 +11,15 @@ use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Authorization as AuthorizationException;
use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Locale\Locale;
use Utopia\Migration\Destination;
use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite;
@ -223,7 +226,7 @@ class Migrations extends Action
}
/**
* @throws Authorization
* @throws AuthorizationException
* @throws Structure
* @throws Conflict
* @throws \Utopia\Database\Exception
@ -282,7 +285,7 @@ class Migrations extends Action
}
/**
* @throws Authorization
* @throws AuthorizationException
* @throws Conflict
* @throws Restricted
* @throws Structure
@ -421,7 +424,7 @@ class Migrations extends Action
* @param Document $migration
* @param Mail $queueForMails
* @return void
* @throws Authorization
* @throws AuthorizationException
* @throws Structure
* @throws \Utopia\Database\Exception
* @throws Exception
@ -432,13 +435,20 @@ class Migrations extends Action
Mail $queueForMails
): void {
$options = $migration->getAttribute('options', []);
$bucketId = $options['bucketId'] ?? null;
$bucketId = 'default'; // Always use platform default bucket
$filename = $options['filename'] ?? 'export_' . \time();
$userInternalId = $options['userInternalId'] ?? '';
$user = $this->dbForPlatform->findOne('users', [
Query::equal('$sequence', [$userInternalId])
]);
$bucket = $this->dbForProject->getDocument('buckets', $bucketId);
if ($user->isEmpty()) {
throw new \Exception('User ' . $userInternalId . ' not found');
}
$bucket = Authorization::skip(fn () => $this->dbForPlatform->getDocument('buckets', $bucketId));
if ($bucket->isEmpty()) {
throw new \Exception("Bucket not found: $bucketId");
throw new \Exception('Bucket not found');
}
$path = $this->deviceForFiles->getPath($bucketId . '/' . $this->sanitizeFilename($filename) . '.csv');
@ -469,7 +479,7 @@ class Migrations extends Action
$this->sendCSVEmail(
success: false,
project: $project,
userInternalId: $userInternalId,
user: $user,
options: $options,
queueForMails: $queueForMails,
sizeMB: $sizeMB
@ -479,9 +489,11 @@ class Migrations extends Action
}
}
$this->dbForProject->createDocument('bucket_' . $bucket->getSequence(), new Document([
$this->dbForPlatform->createDocument('bucket_' . $bucket->getSequence(), new Document([
'$id' => $fileId,
'$permissions' => [],
'$permissions' => [
Permission::read(Role::user($user->getId())),
],
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getSequence(),
'name' => $filename,
@ -511,6 +523,7 @@ class Migrations extends Action
'bucketId' => $bucketId,
'fileId' => $fileId,
'projectId' => $project->getId(),
'internal' => true,
]);
// Generate download URL with JWT
@ -521,7 +534,7 @@ class Migrations extends Action
$this->sendCSVEmail(
success: true,
project: $project,
userInternalId: $userInternalId,
user: $user,
options: $options,
queueForMails: $queueForMails,
downloadUrl: $downloadUrl
@ -533,7 +546,7 @@ class Migrations extends Action
*
* @param bool $success Whether the export was successful
* @param Document $project
* @param string $userInternalId Internal ID of the user
* @param Document $user The user who triggered the operation
* @param array $options Migration options
* @param Mail $queueForMails
* @param string $downloadUrl Download URL for successful exports
@ -544,7 +557,7 @@ class Migrations extends Action
protected function sendCSVEmail(
bool $success,
Document $project,
string $userInternalId,
Document $user,
array $options,
Mail $queueForMails,
string $downloadUrl = '',
@ -554,12 +567,8 @@ class Migrations extends Action
return;
}
$user = $this->dbForPlatform->findOne('users', [
Query::equal('$sequence', [$userInternalId])
]);
if ($user->isEmpty()) {
Console::warning("User not found for CSV export notification: $userInternalId");
Console::warning("User not found for CSV export notification: {$user->getInternalId()}");
return;
}

View file

@ -1282,39 +1282,11 @@ trait MigrationsBase
$this->assertEquals(200, $docs['headers']['status-code']);
$this->assertEquals(10, $docs['body']['total'], 'Expected 10 documents but got ' . $docs['body']['total']);
// Create a storage bucket for the export
$bucketIdUnique = ID::unique();
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'bucketId' => $bucketIdUnique,
'name' => 'Test Export Bucket',
'permissions' => [
Permission::read(Role::any()),
Permission::create(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'fileSecurity' => false,
'enabled' => true,
'maximumFileSize' => 10485760, // 10MB
'allowedFileExtensions' => ['csv'],
'compression' => 'none',
'encryption' => false,
'antivirus' => false
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$bucketId = $bucket['body']['$id'];
// Perform CSV export with notification enabled
// Perform CSV export with notification enabled (uses internal bucket)
$migration = $this->client->call(Client::METHOD_POST, '/migrations/csv/exports', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
'filename' => 'test-export',
'columns' => [],
@ -1329,7 +1301,7 @@ trait MigrationsBase
$this->assertNotEmpty($migration['body']['$id']);
$migrationId = $migration['body']['$id'];
$this->assertEventually(function () use ($bucketId, $migrationId) {
$this->assertEventually(function () use ($migrationId) {
$response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@ -1341,55 +1313,10 @@ trait MigrationsBase
$this->assertEquals('completed', $response['body']['status']);
$this->assertEquals('Appwrite', $response['body']['source']);
$this->assertEquals('CSV', $response['body']['destination']);
$this->assertEquals($bucketId, $response['body']['options']['bucketId']);
return true;
}, 30000, 500);
// Check that the file was created in the bucket
// Query files by filename
$files = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'queries' => [
Query::equal('name', ['test-export'])->toString()
]
]);
$this->assertEquals(200, $files['headers']['status-code']);
$this->assertEquals(1, $files['body']['total'], 'Expected exactly one file with name "test-export"');
// Get the exported file
$file = $files['body']['files'][0];
$fileId = $file['$id'];
$this->assertEquals($bucketId, $file['bucketId']);
$this->assertEquals('test-export', $file['name']);
$this->assertEquals('text/csv', $file['mimeType']);
$this->assertGreaterThan(0, $file['sizeOriginal']);
// Download and verify CSV content
$download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', \array_merge([
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(200, $download['headers']['status-code']);
$csvContent = $download['body'];
$lines = explode("\n", trim($csvContent));
$this->assertCount(11, $lines);
$this->assertStringContainsString('$id', $lines[0]);
$this->assertStringContainsString('$permissions', $lines[0]);
$this->assertStringContainsString('$createdAt', $lines[0]);
$this->assertStringContainsString('$updatedAt', $lines[0]);
$this->assertStringContainsString('name', $lines[0]);
$this->assertStringContainsString('email', $lines[0]);
$this->assertStringContainsString('Test User 1', $lines[1]);
$this->assertStringContainsString('user1@appwrite.io', $lines[1]);
// Check that email was sent with download link
$lastEmail = $this->getLastEmail();
$this->assertNotEmpty($lastEmail);
@ -1407,28 +1334,25 @@ trait MigrationsBase
\parse_str($components['query'] ?? '', $queryParams);
$this->assertArrayHasKey('jwt', $queryParams, 'JWT not found in download URL');
$this->assertNotEmpty($queryParams['jwt']);
$this->assertArrayHasKey('project', $queryParams, 'Project not found in download URL');
$this->assertStringContainsString('/storage/buckets/default/files/', $downloadUrl);
// Test download with JWT
$path = \str_replace('/v1', '', $components['path']);
$downloadWithJwt = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']);
$this->assertEquals(200, $downloadWithJwt['headers']['status-code'], 'Failed to download file with JWT');
$this->assertEquals($csvContent, $downloadWithJwt['body'], 'Downloaded content differs from original');
// Test that download without JWT fails
$downloadWithoutJwt = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download');
$this->assertEquals(404, $downloadWithoutJwt['headers']['status-code'], 'File should not be downloadable without JWT');
// Verify the downloaded content is valid CSV
$csvData = $downloadWithJwt['body'];
$this->assertNotEmpty($csvData, 'CSV export should not be empty');
$this->assertStringContainsString('name', $csvData, 'CSV should contain the name column header');
$this->assertStringContainsString('email', $csvData, 'CSV should contain the email column header');
$this->assertStringContainsString('Test User 1', $csvData, 'CSV should contain test data');
$this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]);
// Cleanup
$this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]);
$this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]);
}
}