diff --git a/.env b/.env index e849e83801..88dec63b1c 100644 --- a/.env +++ b/.env @@ -101,6 +101,7 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 +_APP_INTERVAL_DOMAIN_VERIFICATION=60 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index e146008222..ecc5112cc4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ + chmod +x /usr/local/bin/interval && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php deleted file mode 100644 index ec4cc25ea3..0000000000 --- a/app/controllers/api/storage.php +++ /dev/null @@ -1,2052 +0,0 @@ -desc('Create bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].create') - ->label('audits.event', 'bucket.create') - ->label('audits.resource', 'bucket/{response.$id}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'createBucket', - description: '/docs/references/storage/create-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_CREATED, - model: Response::MODEL_BUCKET, - ) - ] - )) - ->param('bucketId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') - ->param('name', '', new Text(128), 'Bucket name') - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) - ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) - ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) - ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) - ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) - ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) - ->inject('response') - ->inject('dbForProject') - ->inject('queueForEvents') - ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, ?string $compression, ?bool $encryption, bool $antivirus, bool $transformations, Response $response, Database $dbForProject, Event $queueForEvents) { - - $bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId; - - // Map aggregate permissions into the multiple permissions they represent. - $permissions = Permission::aggregate($permissions) ?? []; - $compression ??= Compression::NONE; - $encryption ??= true; - try { - $files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? []; - if (empty($files)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Files collection is not configured.'); - } - - $attributes = []; - $indexes = []; - - foreach ($files['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => $attribute['$id'], - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } - - foreach ($files['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => $index['$id'], - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } - - $dbForProject->createDocument('buckets', new Document([ - '$id' => $bucketId, - '$collection' => 'buckets', - '$permissions' => $permissions, - 'name' => $name, - 'maximumFileSize' => $maximumFileSize, - 'allowedFileExtensions' => $allowedFileExtensions, - 'fileSecurity' => $fileSecurity, - 'enabled' => $enabled, - 'compression' => $compression, - 'encryption' => $encryption, - 'antivirus' => $antivirus, - 'transformations' => $transformations, - 'search' => implode(' ', [$bucketId, $name]), - ])); - - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - $dbForProject->createCollection('bucket_' . $bucket->getSequence(), $attributes, $indexes, permissions: $permissions, documentSecurity: $fileSecurity); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_BUCKET_ALREADY_EXISTS); - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ; - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($bucket, Response::MODEL_BUCKET); - }); - -App::get('/v1/storage/buckets') - ->desc('List buckets') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'listBuckets', - description: '/docs/references/storage/list-buckets.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUCKET_LIST, - ) - ] - )) - ->param('queries', [], new Buckets(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Buckets::ALLOWED_ATTRIBUTES), true) - ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) - ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) - ->inject('response') - ->inject('dbForProject') - ->action(function (array $queries, string $search, bool $includeTotal, Response $response, Database $dbForProject) { - - try { - $queries = Query::parseQueries($queries); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); - } - - if (!empty($search)) { - $queries[] = Query::search('search', $search); - } - - /** - * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries - */ - $cursor = \array_filter($queries, function ($query) { - return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); - }); - $cursor = reset($cursor); - if ($cursor) { - /** @var Query $cursor */ - - $validator = new Cursor(); - if (!$validator->isValid($cursor)) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); - } - - $bucketId = $cursor->getValue(); - $cursorDocument = $dbForProject->getDocument('buckets', $bucketId); - - if ($cursorDocument->isEmpty()) { - throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Bucket '{$bucketId}' for the 'cursor' value not found."); - } - - $cursor->setValue($cursorDocument); - } - - $filterQueries = Query::groupByType($queries)['filters']; - try { - $buckets = $dbForProject->find('buckets', $queries); - $total = $includeTotal ? $dbForProject->count('buckets', $filterQueries, APP_LIMIT_COUNT) : 0; - } catch (OrderException $e) { - throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); - } - $response->dynamic(new Document([ - 'buckets' => $buckets, - 'total' => $total, - ]), Response::MODEL_BUCKET_LIST); - }); - -App::get('/v1/storage/buckets/:bucketId') - ->desc('Get bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'getBucket', - description: '/docs/references/storage/get-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUCKET, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->inject('response') - ->inject('dbForProject') - ->action(function (string $bucketId, Response $response, Database $dbForProject) { - - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $response->dynamic($bucket, Response::MODEL_BUCKET); - }); - -App::put('/v1/storage/buckets/:bucketId') - ->desc('Update bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].update') - ->label('audits.event', 'bucket.update') - ->label('audits.resource', 'bucket/{response.$id}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'updateBucket', - description: '/docs/references/storage/update-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUCKET, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->param('name', null, new Text(128), 'Bucket name', false) - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) - ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) - ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) - ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) - ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) - ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) - ->inject('response') - ->inject('dbForProject') - ->inject('queueForEvents') - ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, ?string $compression, ?bool $encryption, bool $antivirus, bool $transformations, Response $response, Database $dbForProject, Event $queueForEvents) { - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $permissions ??= $bucket->getPermissions(); - $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int) System::getEnv('_APP_STORAGE_LIMIT', 0)); - $allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []); - $enabled ??= $bucket->getAttribute('enabled', true); - $encryption ??= $bucket->getAttribute('encryption', true); - $antivirus ??= $bucket->getAttribute('antivirus', true); - $compression ??= $bucket->getAttribute('compression', Compression::NONE); - $transformations ??= $bucket->getAttribute('transformations', true); - - // Map aggregate permissions into the multiple permissions they represent. - $permissions = Permission::aggregate($permissions); - - $bucket = $dbForProject->updateDocument('buckets', $bucket->getId(), $bucket - ->setAttribute('name', $name) - ->setAttribute('$permissions', $permissions) - ->setAttribute('maximumFileSize', $maximumFileSize) - ->setAttribute('allowedFileExtensions', $allowedFileExtensions) - ->setAttribute('fileSecurity', $fileSecurity) - ->setAttribute('enabled', $enabled) - ->setAttribute('encryption', $encryption) - ->setAttribute('compression', $compression) - ->setAttribute('antivirus', $antivirus) - ->setAttribute('transformations', $transformations)); - - $dbForProject->updateCollection('bucket_' . $bucket->getSequence(), $permissions, $fileSecurity); - - $queueForEvents - ->setParam('bucketId', $bucket->getId()); - - $response->dynamic($bucket, Response::MODEL_BUCKET); - }); - -App::delete('/v1/storage/buckets/:bucketId') - ->desc('Delete bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('audits.event', 'bucket.delete') - ->label('event', 'buckets.[bucketId].delete') - ->label('audits.resource', 'bucket/{request.bucketId}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'deleteBucket', - description: '/docs/references/storage/delete-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_NOCONTENT, - model: Response::MODEL_NONE, - ) - ], - contentType: ContentType::NONE - )) - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->inject('response') - ->inject('dbForProject') - ->inject('queueForDeletes') - ->inject('queueForEvents') - ->action(function (string $bucketId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents) { - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - if (!$dbForProject->deleteDocument('buckets', $bucketId)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove bucket from DB'); - } - - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($bucket); - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setPayload($response->output($bucket, Response::MODEL_BUCKET)) - ; - - $response->noContent(); - }); - -App::post('/v1/storage/buckets/:bucketId/files') - ->alias('/v1/storage/files') - ->desc('Create file') - ->groups(['api', 'storage']) - ->label('scope', 'files.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('audits.event', 'file.create') - ->label('event', 'buckets.[bucketId].files.[fileId].create') - ->label('audits.resource', 'file/{response.$id}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'createFile', - description: '/docs/references/storage/create-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_CREATED, - model: Response::MODEL_FILE, - ) - ], - type: MethodType::UPLOAD, - requestType: ContentType::MULTIPART - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new CustomId(), 'File ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') - ->param('file', [], new File(), 'Binary file. Appwrite SDKs provide helpers to handle file input. [Learn about file input](https://appwrite.io/docs/products/storage/upload-download#input-file).', skipValidation: true) - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE])), 'An array of permission strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->inject('request') - ->inject('response') - ->inject('dbForProject') - ->inject('user') - ->inject('queueForEvents') - ->inject('mode') - ->inject('deviceForFiles') - ->inject('deviceForLocal') - ->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode, Device $deviceForFiles, Device $deviceForLocal) { - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $allowedPermissions = [ - Database::PERMISSION_READ, - Database::PERMISSION_UPDATE, - Database::PERMISSION_DELETE, - ]; - - // Map aggregate permissions to into the set of individual permissions they represent. - $permissions = Permission::aggregate($permissions, $allowedPermissions); - - // Add permissions for current the user if none were provided. - if (\is_null($permissions)) { - $permissions = []; - if (!empty($user->getId()) && !$isPrivilegedUser) { - foreach ($allowedPermissions as $permission) { - $permissions[] = (new Permission($permission, 'user', $user->getId()))->toString(); - } - } - } - - // Users can only manage their own roles, API keys and Admin users can manage any - $roles = Authorization::getRoles(); - if (!$isAPIKey && !$isPrivilegedUser) { - foreach (Database::PERMISSIONS as $type) { - foreach ($permissions as $permission) { - $permission = Permission::parse($permission); - if ($permission->getPermission() != $type) { - continue; - } - $role = (new Role( - $permission->getRole(), - $permission->getIdentifier(), - $permission->getDimension() - ))->toString(); - if (!Authorization::isRole($role)) { - throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); - } - } - } - } - - $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); - if ($maximumFileSize > (int) System::getEnv('_APP_STORAGE_LIMIT', 0)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Maximum bucket file size is larger than _APP_STORAGE_LIMIT'); - } - - - $file = $request->getFiles('file'); - - // GraphQL multipart spec adds files with index keys - if (empty($file)) { - $file = $request->getFiles(0); - } - - if (empty($file)) { - throw new Exception(Exception::STORAGE_FILE_EMPTY); - } - - // Make sure we handle a single file and multiple files the same way - $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - - $contentRange = $request->getHeader('content-range'); - $fileId = $fileId === 'unique()' ? ID::unique() : $fileId; - $chunk = 1; - $chunks = 1; - - if (!empty($contentRange)) { - $start = $request->getContentRangeStart(); - $end = $request->getContentRangeEnd(); - $fileSize = $request->getContentRangeSize(); - $fileId = $request->getHeader('x-appwrite-id', $fileId); - // TODO make `end >= $fileSize` in next breaking version - if (is_null($start) || is_null($end) || is_null($fileSize) || $end > $fileSize) { - throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE); - } - - $idValidator = new UID(); - if (!$idValidator->isValid($fileId)) { - throw new Exception(Exception::STORAGE_INVALID_APPWRITE_ID); - } - - // TODO remove the condition that checks `$end === $fileSize` in next breaking version - if ($end === $fileSize - 1 || $end === $fileSize) { - //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk - $chunks = $chunk = -1; - } else { - // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) - $chunks = (int) ceil($fileSize / ($end + 1 - $start)); - $chunk = (int) ($start / ($end + 1 - $start)) + 1; - } - } - - /** - * Validators - */ - // Check if file type is allowed - $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); - $fileExt = new FileExt($allowedFileExtensions); - if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { - throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, 'File extension not allowed'); - } - - // Check if file size is exceeding allowed limit - $fileSizeValidator = new FileSize($maximumFileSize); - if (!$fileSizeValidator->isValid($fileSize)) { - throw new Exception(Exception::STORAGE_INVALID_FILE_SIZE, 'File size not allowed'); - } - - $upload = new Upload(); - if (!$upload->isValid($fileTmpName)) { - throw new Exception(Exception::STORAGE_INVALID_FILE); - } - - // Save to storage - $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); - $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); - $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root - - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - - $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; - if (!$file->isEmpty()) { - $chunks = $file->getAttribute('chunksTotal', 1); - $uploaded = $file->getAttribute('chunksUploaded', 0); - $metadata = $file->getAttribute('metadata', []); - - if ($chunk === -1) { - $chunk = $chunks; - } - - if ($uploaded === $chunks) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } - } - - $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); - - if (empty($chunksUploaded)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); - } - - if ($chunksUploaded === $chunks) { - if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { - $antivirus = new Network( - System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), - (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) - ); - - if (!$antivirus->fileScan($path)) { - $deviceForFiles->delete($path); - throw new Exception(Exception::STORAGE_INVALID_FILE); - } - } - - $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption - $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption - $data = ''; - // Compression - $algorithm = $bucket->getAttribute('compression', Compression::NONE); - if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { - $data = $deviceForFiles->read($path); - switch ($algorithm) { - case Compression::ZSTD: - $compressor = new Zstd(); - break; - case Compression::GZIP: - default: - $compressor = new GZIP(); - break; - } - $data = $compressor->compress($data); - } else { - // reset the algorithm to none as we do not compress the file - // if file size exceedes the APP_STORAGE_READ_BUFFER - // regardless the bucket compression algoorithm - $algorithm = Compression::NONE; - } - - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - if (empty($data)) { - $data = $deviceForFiles->read($path); - } - $key = System::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); - } - - if (!empty($data)) { - if (!$deviceForFiles->write($path, $data, $mimeType)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); - } - } - - $sizeActual = $deviceForFiles->getFileSize($path); - - $openSSLVersion = null; - $openSSLCipher = null; - $openSSLTag = null; - $openSSLIV = null; - - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - $openSSLVersion = '1'; - $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; - $openSSLTag = \bin2hex($tag); - $openSSLIV = \bin2hex($iv); - } - - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => $fileId, - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => $fileHash, - 'mimeType' => $mimeType, - 'sizeOriginal' => $fileSize, - 'sizeActual' => $sizeActual, - 'algorithm' => $algorithm, - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'openSSLVersion' => $openSSLVersion, - 'openSSLCipher' => $openSSLCipher, - 'openSSLTag' => $openSSLTag, - 'openSSLIV' => $openSSLIV, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - $file = $file - ->setAttribute('$permissions', $permissions) - ->setAttribute('signature', $fileHash) - ->setAttribute('mimeType', $mimeType) - ->setAttribute('sizeActual', $sizeActual) - ->setAttribute('algorithm', $algorithm) - ->setAttribute('openSSLVersion', $openSSLVersion) - ->setAttribute('openSSLCipher', $openSSLCipher) - ->setAttribute('openSSLTag', $openSSLTag) - ->setAttribute('openSSLIV', $openSSLIV) - ->setAttribute('metadata', $metadata) - ->setAttribute('chunksUploaded', $chunksUploaded); - - /** - * Validate create permission and skip authorization in updateDocument - * Without this, the file creation will fail when user doesn't have update permission - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we validate create permission instead of update - */ - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } - } else { - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => ID::custom($fileId), - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => '', - 'mimeType' => '', - 'sizeOriginal' => $fileSize, - 'sizeActual' => 0, - 'algorithm' => '', - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - $file = $file - ->setAttribute('chunksUploaded', $chunksUploaded) - ->setAttribute('metadata', $metadata); - - /** - * Validate create permission and skip authorization in updateDocument - * Without this, the file creation will fail when user doesn't have update permission - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we validate create permission instead of update - */ - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - try { - $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket); - - $metadata = null; // was causing leaks as it was passed by reference - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($file, Response::MODEL_FILE); - }); - -App::get('/v1/storage/buckets/:bucketId/files') - ->alias('/v1/storage/files') - ->desc('List files') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'listFiles', - description: '/docs/references/storage/list-files.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_FILE_LIST, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('queries', [], new Files(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Files::ALLOWED_ATTRIBUTES), true) - ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) - ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) - ->inject('response') - ->inject('dbForProject') - ->inject('mode') - ->action(function (string $bucketId, array $queries, string $search, bool $includeTotal, Response $response, Database $dbForProject, string $mode) { - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $queries = Query::parseQueries($queries); - - if (!empty($search)) { - $queries[] = Query::search('search', $search); - } - - /** - * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries - */ - $cursor = \array_filter($queries, function ($query) { - return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); - }); - $cursor = reset($cursor); - if ($cursor) { - /** @var Query $cursor */ - - $validator = new Cursor(); - if (!$validator->isValid($cursor)) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); - } - - $fileId = $cursor->getValue(); - - if ($fileSecurity && !$valid) { - $cursorDocument = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if ($cursorDocument->isEmpty()) { - throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "File '{$fileId}' for the 'cursor' value not found."); - } - - $cursor->setValue($cursorDocument); - } - - $filterQueries = Query::groupByType($queries)['filters']; - - try { - if ($fileSecurity && !$valid) { - $files = $dbForProject->find('bucket_' . $bucket->getSequence(), $queries); - $total = $includeTotal ? $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT) : 0; - } else { - $files = Authorization::skip(fn () => $dbForProject->find('bucket_' . $bucket->getSequence(), $queries)); - $total = $includeTotal ? Authorization::skip(fn () => $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT)) : 0; - } - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } catch (OrderException $e) { - throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); - } - - $response->dynamic(new Document([ - 'files' => $files, - 'total' => $total, - ]), Response::MODEL_FILE_LIST); - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId') - ->alias('/v1/storage/files/:fileId') - ->desc('Get file') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFile', - description: '/docs/references/storage/get-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_FILE, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - ->inject('response') - ->inject('dbForProject') - ->inject('mode') - ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, string $mode) { - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $response->dynamic($file, Response::MODEL_FILE); - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') - ->alias('/v1/storage/files/:fileId/preview') - ->desc('Get file preview') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('cache', true) - ->label('cache.resourceType', 'bucket/{request.bucketId}') - ->label('cache.resource', 'file/{request.fileId}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFilePreview', - description: '/docs/references/storage/get-file-preview.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_NONE - ) - ], - type: MethodType::LOCATION, - contentType: ContentType::IMAGE - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID') - ->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true) - ->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true) - ->param('gravity', Image::GRAVITY_CENTER, new WhiteList(Image::getGravityTypes()), 'Image crop gravity. Can be one of ' . implode(",", Image::getGravityTypes()), true) - ->param('quality', -1, new Range(-1, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to keep existing image quality.', true) - ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) - ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) - ->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true) - ->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) - ->param('rotation', 0, new Range(-360, 360), 'Preview image rotation in degrees. Pass an integer between -360 and 360.', true) - ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) - ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) - // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. - ->param('token', '', new Text(512), 'File token for accessing this file.', true) - ->inject('request') - ->inject('response') - ->inject('dbForProject') - ->inject('resourceToken') - ->inject('deviceForFiles') - ->inject('deviceForLocal') - ->inject('project') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, ?string $token, Request $request, Response $response, Database $dbForProject, Document $resourceToken, Device $deviceForFiles, Device $deviceForLocal, Document $project) { - - if (!\extension_loaded('imagick')) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); - } - - /* @type Document $bucket */ - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - if (!$bucket->getAttribute('transformations', true) && !$isAPIKey && !$isPrivilegedUser) { - throw new Exception(Exception::STORAGE_BUCKET_TRANSFORMATIONS_DISABLED); - } - - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid && !$isToken) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid && !$isToken) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - /* @type Document $file */ - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $inputs = Config::getParam('storage-inputs'); - $outputs = Config::getParam('storage-outputs'); - $fileLogos = Config::getParam('storage-logos'); - - $path = $file->getAttribute('path'); - $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $algorithm = $file->getAttribute('algorithm', Compression::NONE); - $cipher = $file->getAttribute('openSSLCipher'); - $mime = $file->getAttribute('mimeType'); - if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) System::getEnv('_APP_STORAGE_PREVIEW_LIMIT', APP_STORAGE_READ_BUFFER)) { - if (!\in_array($mime, $inputs)) { - $path = (\array_key_exists($mime, $fileLogos)) ? $fileLogos[$mime] : $fileLogos['default']; - } else { - // it was an image but the file size exceeded the limit - $path = $fileLogos['default_image']; - } - - $algorithm = Compression::NONE; - $cipher = null; - $background = (empty($background)) ? 'eceff1' : $background; - $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $deviceForFiles = $deviceForLocal; - } - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - if (empty($output)) { - // when file extension is provided but it's not one of our - // supported outputs we fallback to `jpg` - if (!empty($type) && !array_key_exists($type, $outputs)) { - $type = 'jpg'; - } - - // when file extension is not provided and the mime type is not one of our supported outputs - // we fallback to `jpg` output format - $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; - } - - $startTime = \microtime(true); - - $source = $deviceForFiles->read($path); - - $downloadTime = \microtime(true) - $startTime; - - if (!empty($cipher)) { // Decrypt - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - $decryptionTime = \microtime(true) - $startTime - $downloadTime; - - switch ($algorithm) { - case Compression::ZSTD: - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - $decompressionTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime; - - try { - $image = new Image($source); - } catch (ImagickException $e) { - throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, $e->getMessage()); - } - - $image->crop((int) $width, (int) $height, $gravity); - - if (!empty($opacity) || $opacity === 0) { - $image->setOpacity($opacity); - } - - if (!empty($background)) { - $image->setBackground('#' . $background); - } - - if (!empty($borderWidth)) { - $image->setBorder($borderWidth, '#' . $borderColor); - } - - if (!empty($borderRadius)) { - $image->setBorderRadius($borderRadius); - } - - if (!empty($rotation)) { - $image->setRotation(($rotation + 360) % 360); - } - - $data = $image->output($output, $quality); - - $renderingTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime - $decompressionTime; - - $totalTime = \microtime(true) - $startTime; - - Console::info("File preview rendered,project=" . $project->getId() . ",bucket=" . $bucketId . ",file=" . $file->getId() . ",uri=" . $request->getURI() . ",total=" . $totalTime . ",rendering=" . $renderingTime . ",decryption=" . $decryptionTime . ",decompression=" . $decompressionTime . ",download=" . $downloadTime); - - $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - - //Do not update transformedAt if it's a console user - if (!User::isPrivileged(Authorization::getRoles())) { - $transformedAt = $file->getAttribute('transformedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { - $file->setAttribute('transformedAt', DateTime::now()); - Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); - } - } - - $response - ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days - ->setContentType($contentType) - ->file($data); - - unset($image); - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') - ->alias('/v1/storage/files/:fileId/download') - ->desc('Get file for download') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFileDownload', - description: '/docs/references/storage/get-file-download.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_NONE - ) - ], - type: MethodType::LOCATION, - contentType: ContentType::ANY, - )) - ->param('bucketId', '', new UID(), 'Storage bucket ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. - ->param('token', '', new Text(512), 'File token for accessing this file.', true) - ->inject('request') - ->inject('response') - ->inject('dbForProject') - ->inject('mode') - ->inject('resourceToken') - ->inject('deviceForFiles') - ->action(function (string $bucketId, string $fileId, ?string $token, Request $request, Response $response, Database $dbForProject, string $mode, Document $resourceToken, Device $deviceForFiles) { - /* @type Document $bucket */ - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid && !$isToken) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid && !$isToken) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - /* @type Document $file */ - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $path = $file->getAttribute('path', ''); - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); - } - - $size = $file->getAttribute('sizeOriginal', 0); - - $rangeHeader = $request->getHeader('range'); - if (!empty($rangeHeader)) { - $start = $request->getRangeStart(); - $end = $request->getRangeEnd(); - $unit = $request->getRangeUnit(); - - if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { - $end = min(($start + MAX_OUTPUT_CHUNK_SIZE - 1), ($size - 1)); - } - - if ($unit !== 'bytes' || $start >= $end || $end >= $size) { - throw new Exception(Exception::STORAGE_INVALID_RANGE); - } - - $response - ->addHeader('Accept-Ranges', 'bytes') - ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) - ->addHeader('Content-Length', $end - $start + 1) - ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - } - - $response - ->setContentType($file->getAttribute('mimeType')) - ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days - ->addHeader('X-Peak', \memory_get_peak_usage()) - ->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"') - ; - - $source = ''; - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceForFiles->read($path); - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - if (!empty($source)) { - if (!empty($rangeHeader)) { - $response->send(substr($source, $start, ($end - $start + 1))); - return; - } - $response->send($source); - return; - } - - if (!empty($rangeHeader)) { - $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); - return; - } - - if ($size > APP_STORAGE_READ_BUFFER) { - for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { - $response->chunk( - $deviceForFiles->read( - $path, - ($i * MAX_OUTPUT_CHUNK_SIZE), - min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) - ), - (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size - ); - } - } else { - $response->send($deviceForFiles->read($path)); - } - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') - ->alias('/v1/storage/files/:fileId/view') - ->desc('Get file for view') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFileView', - description: '/docs/references/storage/get-file-view.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_NONE, - ) - ], - type: MethodType::LOCATION, - contentType: ContentType::ANY, - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. - ->param('token', '', new Text(512), 'File token for accessing this file.', true) - ->inject('response') - ->inject('request') - ->inject('dbForProject') - ->inject('mode') - ->inject('resourceToken') - ->inject('deviceForFiles') - ->action(function (string $bucketId, string $fileId, ?string $token, Response $response, Request $request, Database $dbForProject, string $mode, Document $resourceToken, Device $deviceForFiles) { - /* @type Document $bucket */ - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid && !$isToken) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid && !$isToken) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - /* @type Document $file */ - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $mimes = Config::getParam('storage-mimes'); - - $path = $file->getAttribute('path', ''); - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); - } - - $contentType = 'text/plain'; - - if (\in_array($file->getAttribute('mimeType'), $mimes)) { - $contentType = $file->getAttribute('mimeType'); - } - - $size = $file->getAttribute('sizeOriginal', 0); - - $rangeHeader = $request->getHeader('range'); - if (!empty($rangeHeader)) { - $start = $request->getRangeStart(); - $end = $request->getRangeEnd(); - $unit = $request->getRangeUnit(); - - if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { - $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); - } - - if ($unit != 'bytes' || $start >= $end || $end >= $size) { - throw new Exception(Exception::STORAGE_INVALID_RANGE); - } - - $response - ->addHeader('Accept-Ranges', 'bytes') - ->addHeader('Content-Range', "bytes $start-$end/$size") - ->addHeader('Content-Length', $end - $start + 1) - ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - } - - $response - ->setContentType($contentType) - ->addHeader('Content-Security-Policy', 'script-src none;') - ->addHeader('X-Content-Type-Options', 'nosniff') - ->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"') - ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days - ->addHeader('X-Peak', \memory_get_peak_usage()) - ; - - $source = ''; - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceForFiles->read($path); - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - if (!empty($source)) { - if (!empty($rangeHeader)) { - $response->send(substr($source, $start, ($end - $start + 1))); - return; - } - $response->send($source); - return; - } - - if (!empty($rangeHeader)) { - $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); - return; - } - - $size = $deviceForFiles->getFileSize($path); - if ($size > APP_STORAGE_READ_BUFFER) { - for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { - $response->chunk( - $deviceForFiles->read( - $path, - ($i * MAX_OUTPUT_CHUNK_SIZE), - min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) - ), - (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size - ); - } - } else { - $response->send($deviceForFiles->read($path)); - } - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/push') - ->desc('Get file for push notification') - ->groups(['api', 'storage']) - ->label('scope', 'public') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - ->param('jwt', '', new Text(2048, 0), 'JSON Web Token to validate', true) - ->inject('response') - ->inject('request') - ->inject('dbForProject') - ->inject('dbForPlatform') - ->inject('project') - ->inject('mode') - ->inject('deviceForFiles') - ->action(function (string $bucketId, string $fileId, string $jwt, Response $response, Request $request, Database $dbForProject, Database $dbForPlatform, Document $project, string $mode, Device $deviceForFiles) { - $decoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); - - try { - $decoded = $decoder->decode($jwt); - } catch (JWTException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ( - $decoded['projectId'] !== $project->getId() || - $decoded['bucketId'] !== $bucketId || - $decoded['fileId'] !== $fileId - ) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $isInternal = $decoded['internal'] ?? false; - $disposition = $decoded['disposition'] ?? 'inline'; - $dbForProject = $isInternal ? $dbForPlatform : $dbForProject; - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $mimes = Config::getParam('storage-mimes'); - - $path = $file->getAttribute('path', ''); - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); - } - - $contentType = 'text/plain'; - - if (\in_array($file->getAttribute('mimeType'), $mimes)) { - $contentType = $file->getAttribute('mimeType'); - } - - $size = $file->getAttribute('sizeOriginal', 0); - - $rangeHeader = $request->getHeader('range'); - if (!empty($rangeHeader)) { - $start = $request->getRangeStart(); - $end = $request->getRangeEnd(); - $unit = $request->getRangeUnit(); - - if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { - $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); - } - - if ($unit != 'bytes' || $start >= $end || $end >= $size) { - throw new Exception(Exception::STORAGE_INVALID_RANGE); - } - - $response - ->addHeader('Accept-Ranges', 'bytes') - ->addHeader('Content-Range', "bytes $start-$end/$size") - ->addHeader('Content-Length', $end - $start + 1) - ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - } - - $response - ->setContentType($contentType) - ->addHeader('Content-Security-Policy', 'script-src none;') - ->addHeader('X-Content-Type-Options', 'nosniff') - ->addHeader('Content-Disposition', $disposition . '; filename="' . $file->getAttribute('name', '') . '"') - ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days - ->addHeader('X-Peak', \memory_get_peak_usage()); - - $source = ''; - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceForFiles->read($path); - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - if (!empty($source)) { - if (!empty($rangeHeader)) { - $response->send(substr($source, $start, ($end - $start + 1))); - return; - } - $response->send($source); - return; - } - - if (!empty($rangeHeader)) { - $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); - return; - } - - $size = $deviceForFiles->getFileSize($path); - if ($size > APP_STORAGE_READ_BUFFER) { - for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { - $response->chunk( - $deviceForFiles->read( - $path, - ($i * MAX_OUTPUT_CHUNK_SIZE), - min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) - ), - (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size - ); - } - } else { - $response->send($deviceForFiles->read($path)); - } - }); - -App::put('/v1/storage/buckets/:bucketId/files/:fileId') - ->alias('/v1/storage/files/:fileId') - ->desc('Update file') - ->groups(['api', 'storage']) - ->label('scope', 'files.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].files.[fileId].update') - ->label('audits.event', 'file.update') - ->label('audits.resource', 'file/{response.$id}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'updateFile', - description: '/docs/references/storage/update-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_FILE, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File unique ID.') - ->param('name', null, new Nullable(new Text(255)), 'Name of the file', true) - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE])), 'An array of permission string. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->inject('response') - ->inject('dbForProject') - ->inject('user') - ->inject('mode') - ->inject('queueForEvents') - ->action(function (string $bucketId, string $fileId, ?string $name, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $queueForEvents) { - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_UPDATE); - $valid = $validator->isValid($bucket->getUpdate()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - // Read permission should not be required for update - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - // Map aggregate permissions into the multiple permissions they represent. - $permissions = Permission::aggregate($permissions, [ - Database::PERMISSION_READ, - Database::PERMISSION_UPDATE, - Database::PERMISSION_DELETE, - ]); - - // Users can only manage their own roles, API keys and Admin users can manage any - $roles = Authorization::getRoles(); - if (!User::isApp($roles) && !User::isPrivileged($roles) && !\is_null($permissions)) { - foreach (Database::PERMISSIONS as $type) { - foreach ($permissions as $permission) { - $permission = Permission::parse($permission); - if ($permission->getPermission() != $type) { - continue; - } - $role = (new Role( - $permission->getRole(), - $permission->getIdentifier(), - $permission->getDimension() - ))->toString(); - if (!Authorization::isRole($role)) { - throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); - } - } - } - } - - if (\is_null($permissions)) { - $permissions = $file->getPermissions() ?? []; - } - - $file->setAttribute('$permissions', $permissions); - - if (!is_null($name)) { - $file->setAttribute('name', $name); - } - - try { - if ($fileSecurity && !$valid) { - $file = $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file); - } else { - $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket) - ; - - $response->dynamic($file, Response::MODEL_FILE); - }); - -App::delete('/v1/storage/buckets/:bucketId/files/:fileId') - ->desc('Delete file') - ->groups(['api', 'storage']) - ->label('scope', 'files.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].files.[fileId].delete') - ->label('audits.event', 'file.delete') - ->label('audits.resource', 'file/{request.fileId}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'deleteFile', - description: '/docs/references/storage/delete-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_NOCONTENT, - model: Response::MODEL_NONE, - ) - ], - contentType: ContentType::NONE - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - ->inject('response') - ->inject('dbForProject') - ->inject('queueForEvents') - ->inject('mode') - ->inject('deviceForFiles') - ->inject('queueForDeletes') - ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Device $deviceForFiles, Delete $queueForDeletes) { - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_DELETE); - $valid = $validator->isValid($bucket->getDelete()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - // Read permission should not be required for delete - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - // Make sure we don't delete the file before the document permission check occurs - if ($fileSecurity && !$valid && !$validator->isValid($file->getDelete())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $deviceDeleted = false; - if ($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) { - $deviceDeleted = $deviceForFiles->abort( - $file->getAttribute('path'), - ($file->getAttribute('metadata', [])['uploadId'] ?? '') - ); - } else { - $deviceDeleted = $deviceForFiles->delete($file->getAttribute('path')); - } - - if ($deviceDeleted) { - $queueForDeletes - ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) - ->setResourceType('bucket/' . $bucket->getId()) - ->setResource('file/' . $fileId) - ; - - try { - if ($fileSecurity && !$valid) { - $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - if (!$deleted) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB'); - } - } else { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to delete file from device'); - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket) - ->setPayload($response->output($file, Response::MODEL_FILE)) - ; - - $response->noContent(); - }); - -/** Storage usage */ -App::get('/v1/storage/usage') - ->desc('Get storage usage stats') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: null, - name: 'getUsage', - description: '/docs/references/storage/get-usage.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_USAGE_STORAGE, - ) - ] - )) - ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('dbForProject') - ->action(function (string $range, Response $response, Database $dbForProject) { - - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - METRIC_BUCKETS, - METRIC_FILES, - METRIC_FILES_STORAGE, - ]; - - $total = []; - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { - foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - - $stats[$metric]['total'] = $result['value'] ?? 0; - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', [$period]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric]['data'] = []; - foreach ($results as $result) { - $stats[$metric]['data'][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric]['total'] = $stats[$metric]['total']; - $usage[$metric]['data'] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric]['data'][] = [ - 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, - 'date' => $formatDate, - ]; - } - } - $response->dynamic(new Document([ - 'range' => $range, - 'bucketsTotal' => $usage[$metrics[0]]['total'], - 'filesTotal' => $usage[$metrics[1]]['total'], - 'filesStorageTotal' => $usage[$metrics[2]]['total'], - 'buckets' => $usage[$metrics[0]]['data'], - 'files' => $usage[$metrics[1]]['data'], - 'storage' => $usage[$metrics[2]]['data'], - ]), Response::MODEL_USAGE_STORAGE); - }); - -App::get('/v1/storage/:bucketId/usage') - ->desc('Get bucket usage stats') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: null, - name: 'getBucketUsage', - description: '/docs/references/storage/get-bucket-usage.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_USAGE_BUCKETS, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Bucket ID.') - ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('project') - ->inject('dbForProject') - ->inject('getLogsDB') - ->action(function (string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, callable $getLogsDB) { - - $dbForLogs = call_user_func($getLogsDB, $project); - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES), - str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_STORAGE), - str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED), - ]; - - Authorization::skip(function () use ($dbForProject, $dbForLogs, $bucket, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $db = ($metric === str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED)) - ? $dbForLogs - : $dbForProject; - - $result = $db->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - - $stats[$metric]['total'] = $result['value'] ?? 0; - $limit = $days['limit']; - $period = $days['period']; - $results = $db->find('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', [$period]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric]['data'] = []; - foreach ($results as $result) { - $stats[$metric]['data'][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric]['total'] = $stats[$metric]['total']; - $usage[$metric]['data'] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric]['data'][] = [ - 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, - 'date' => $formatDate, - ]; - } - } - - $response->dynamic(new Document([ - 'range' => $range, - 'filesTotal' => $usage[$metrics[0]]['total'], - 'filesStorageTotal' => $usage[$metrics[1]]['total'], - 'files' => $usage[$metrics[0]]['data'], - 'storage' => $usage[$metrics[1]]['data'], - 'imageTransformations' => $usage[$metrics[2]]['data'], - 'imageTransformationsTotal' => $usage[$metrics[2]]['total'], - ]), Response::MODEL_USAGE_BUCKETS); - }); diff --git a/bin/interval b/bin/interval new file mode 100644 index 0000000000..e4355b1dc3 --- /dev/null +++ b/bin/interval @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php interval $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3b935b84fb..805be67340 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -555,6 +555,7 @@ services: - _APP_DOMAIN_TARGET_CAA - _APP_DNS - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES - _APP_EMAIL_CERTIFICATES - _APP_REDIS_HOST - _APP_REDIS_PORT @@ -753,6 +754,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: + - mariadb - redis environment: - _APP_ENV @@ -785,6 +787,43 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES + appwrite-task-interval: + entrypoint: interval + <<: *x-logging + container_name: appwrite-task-interval + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_DOMAIN + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A + - _APP_DOMAIN_TARGET_CAA + - _APP_DNS + - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_DATABASE_SHARED_TABLES + - _APP_INTERVAL_DOMAIN_VERIFICATION + appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index ab673e9cfe..47d865ad08 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -8,6 +8,10 @@ interface Adapter { public function issueCertificate(string $certName, string $domain, ?string $domainType): ?string; + public function isInstantGeneration(string $domain, ?string $domainType): bool; + + public function getCertificateStatus(string $domain, ?string $domainType): string; + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; public function deleteCertificate(string $domain): void; diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php new file mode 100644 index 0000000000..ca15a95ed8 --- /dev/null +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -0,0 +1,10 @@ +skipRenewCheck; } + /** + * Set action for this certificate event. + * + * @param string $action + * @return self + */ + public function setAction(string $action): self + { + $this->action = $action; + return $this; + } + + /** + * Get action for this certificate event. + * + * @return string + */ + public function getAction(): string + { + return $this->action; + } + /** * Prepare the payload for the event @@ -103,7 +128,8 @@ class Certificate extends Event 'project' => $this->project, 'domain' => $this->domain, 'skipRenewCheck' => $this->skipRenewCheck, - 'validationDomain' => $this->validationDomain + 'validationDomain' => $this->validationDomain, + 'action' => $this->action ]; } } diff --git a/src/Appwrite/Platform/Appwrite.php b/src/Appwrite/Platform/Appwrite.php index 4aa135c4f1..a34c79308a 100644 --- a/src/Appwrite/Platform/Appwrite.php +++ b/src/Appwrite/Platform/Appwrite.php @@ -10,6 +10,7 @@ use Appwrite\Platform\Modules\Functions; use Appwrite\Platform\Modules\Projects; use Appwrite\Platform\Modules\Proxy; use Appwrite\Platform\Modules\Sites; +use Appwrite\Platform\Modules\Storage; use Appwrite\Platform\Modules\Tokens; use Utopia\Platform\Platform; @@ -26,5 +27,6 @@ class Appwrite extends Platform $this->addModule(new Console\Module()); $this->addModule(new Proxy\Module()); $this->addModule(new Tokens\Module()); + $this->addModule(new Storage\Module()); } } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php b/src/Appwrite/Platform/Modules/Proxy/Action.php similarity index 99% rename from src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php rename to src/Appwrite/Platform/Modules/Proxy/Action.php index 5ec5b8b8f5..c3fa535a5c 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php +++ b/src/Appwrite/Platform/Modules/Proxy/Action.php @@ -1,6 +1,6 @@ $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php index 13ee6b9cb5..ea0fb69050 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Function; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php index 4581cb3d08..4c17fdc460 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php index 7ddd501d51..f21374b49a 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Redirect; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; @@ -147,6 +147,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php index 7aff9e7e8a..26bd453eb3 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Site; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php index e09c4a6eba..7266e8d183 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Verification; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php index 198bf55a6f..e160b71060 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php new file mode 100644 index 0000000000..00daef061e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php @@ -0,0 +1,166 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/storage/buckets') + ->desc('Create bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].create') + ->label('audits.event', 'bucket.create') + ->label('audits.resource', 'bucket/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'createBucket', + description: '/docs/references/storage/create-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('name', '', new Text(128), 'Bucket name') + ->param('permissions', null, new Nullable(new \Utopia\Database\Validator\Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) + ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) + ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $name, + ?array $permissions, + bool $fileSecurity, + bool $enabled, + int $maximumFileSize, + array $allowedFileExtensions, + ?string $compression, + ?bool $encryption, + bool $antivirus, + bool $transformations, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId; + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions) ?? []; + $compression ??= Compression::NONE; + $encryption ??= true; + try { + $files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? []; + if (empty($files)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + 'default' => $attribute['default'] ?? null, + 'format' => $attribute['format'] ?? '' + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForProject->createDocument('buckets', new Document([ + '$id' => $bucketId, + '$collection' => 'buckets', + '$permissions' => $permissions, + 'name' => $name, + 'maximumFileSize' => $maximumFileSize, + 'allowedFileExtensions' => $allowedFileExtensions, + 'fileSecurity' => $fileSecurity, + 'enabled' => $enabled, + 'compression' => $compression, + 'encryption' => $encryption, + 'antivirus' => $antivirus, + 'transformations' => $transformations, + 'search' => implode(' ', [$bucketId, $name]), + ])); + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + $dbForProject->createCollection('bucket_' . $bucket->getSequence(), $attributes, $indexes, permissions: $permissions, documentSecurity: $fileSecurity); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_BUCKET_ALREADY_EXISTS); + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php new file mode 100644 index 0000000000..9523f55e12 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php @@ -0,0 +1,89 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Delete bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('audits.event', 'bucket.delete') + ->label('event', 'buckets.[bucketId].delete') + ->label('audits.resource', 'bucket/{request.bucketId}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'deleteBucket', + description: '/docs/references/storage/delete-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_NOCONTENT, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + Response $response, + Database $dbForProject, + DeleteEvent $queueForDeletes, + Event $queueForEvents + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!$dbForProject->deleteDocument('buckets', $bucketId)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove bucket from DB'); + } + + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($bucket); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setPayload($response->output($bucket, Response::MODEL_BUCKET)) + ; + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php new file mode 100644 index 0000000000..b2d9af5a08 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -0,0 +1,451 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/storage/buckets/:bucketId/files') + ->desc('Create file') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('audits.event', 'file.create') + ->label('event', 'buckets.[bucketId].files.[fileId].create') + ->label('audits.resource', 'file/{response.$id}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'createFile', + description: '/docs/references/storage/create-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_FILE, + ) + ], + type: MethodType::UPLOAD, + requestType: ContentType::MULTIPART + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new CustomId(), 'File ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('file', [], new File(), 'Binary file. Appwrite SDKs provide helpers to handle file input. [Learn about file input](https://appwrite.io/docs/products/storage/upload-download#input-file).', skipValidation: true) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE])), 'An array of permission strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('user') + ->inject('queueForEvents') + ->inject('mode') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + mixed $file, + ?array $permissions, + Request $request, + Response $response, + Database $dbForProject, + Document $user, + Event $queueForEvents, + string $mode, + Device $deviceForFiles, + Device $deviceForLocal + ) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $allowedPermissions = [ + \Utopia\Database\Database::PERMISSION_READ, + \Utopia\Database\Database::PERMISSION_UPDATE, + \Utopia\Database\Database::PERMISSION_DELETE, + ]; + + // Map aggregate permissions to into the set of individual permissions they represent. + $permissions = Permission::aggregate($permissions, $allowedPermissions); + + // Add permissions for current the user if none were provided. + if (\is_null($permissions)) { + $permissions = []; + if (!empty($user->getId()) && !$isPrivilegedUser) { + foreach ($allowedPermissions as $permission) { + $permissions[] = (new Permission($permission, 'user', $user->getId()))->toString(); + } + } + } + + // Users can only manage their own roles, API keys and Admin users can manage any + $roles = Authorization::getRoles(); + if (!$isAPIKey && !$isPrivilegedUser) { + foreach (\Utopia\Database\Database::PERMISSIONS as $type) { + foreach ($permissions as $permission) { + $permission = Permission::parse($permission); + if ($permission->getPermission() != $type) { + continue; + } + $role = (new Role( + $permission->getRole(), + $permission->getIdentifier(), + $permission->getDimension() + ))->toString(); + if (!Authorization::isRole($role)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); + } + } + } + } + + $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); + if ($maximumFileSize > (int) System::getEnv('_APP_STORAGE_LIMIT', 0)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Maximum bucket file size is larger than _APP_STORAGE_LIMIT'); + } + + $file = $request->getFiles('file'); + + // GraphQL multipart spec adds files with index keys + if (empty($file)) { + $file = $request->getFiles(0); + } + + if (empty($file)) { + throw new Exception(Exception::STORAGE_FILE_EMPTY); + } + + // Make sure we handle a single file and multiple files the same way + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + + $contentRange = $request->getHeader('content-range'); + $fileId = $fileId === 'unique()' ? ID::unique() : $fileId; + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $fileSize = $request->getContentRangeSize(); + $fileId = $request->getHeader('x-appwrite-id', $fileId); + // TODO make `end >= $fileSize` in next breaking version + if (is_null($start) || is_null($end) || is_null($fileSize) || $end > $fileSize) { + throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE); + } + + $idValidator = new UID(); + if (!$idValidator->isValid($fileId)) { + throw new Exception(Exception::STORAGE_INVALID_APPWRITE_ID); + } + + // TODO remove the condition that checks `$end === $fileSize` in next breaking version + if ($end === $fileSize - 1 || $end === $fileSize) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($fileSize / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)) + 1; + } + } + + /** + * Validators + */ + // Check if file type is allowed + $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); + $fileExt = new FileExt($allowedFileExtensions); + if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, 'File extension not allowed'); + } + + // Check if file size is exceeding allowed limit + $fileSizeValidator = new FileSize($maximumFileSize); + if (!$fileSizeValidator->isValid($fileSize)) { + throw new Exception(Exception::STORAGE_INVALID_FILE_SIZE, 'File size not allowed'); + } + + $upload = new Upload(); + if (!$upload->isValid($fileTmpName)) { + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + + // Save to storage + $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); + $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + + $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + $uploaded = $file->getAttribute('chunksUploaded', 0); + $metadata = $file->getAttribute('metadata', []); + + if ($chunk === -1) { + $chunk = $chunks; + } + + if ($uploaded === $chunks) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } + } + + $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + + if (empty($chunksUploaded)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); + } + + if ($chunksUploaded === $chunks) { + if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { + $antivirus = new Network( + System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) + ); + + if (!$antivirus->fileScan($path)) { + $deviceForFiles->delete($path); + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + } + + $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption + $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption + $data = ''; + // Compression + $algorithm = $bucket->getAttribute('compression', Compression::NONE); + if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { + $data = $deviceForFiles->read($path); + switch ($algorithm) { + case Compression::ZSTD: + $compressor = new Zstd(); + break; + case Compression::GZIP: + default: + $compressor = new GZIP(); + break; + } + $data = $compressor->compress($data); + } else { + // reset the algorithm to none as we do not compress the file + // if file size exceedes the APP_STORAGE_READ_BUFFER + // regardless the bucket compression algoorithm + $algorithm = Compression::NONE; + } + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + if (empty($data)) { + $data = $deviceForFiles->read($path); + } + $key = System::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if (!empty($data)) { + if (!$deviceForFiles->write($path, $data, $mimeType)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); + } + } + + $sizeActual = $deviceForFiles->getFileSize($path); + + $openSSLVersion = null; + $openSSLCipher = null; + $openSSLTag = null; + $openSSLIV = null; + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $fileSize, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + $file = $file + ->setAttribute('$permissions', $permissions) + ->setAttribute('signature', $fileHash) + ->setAttribute('mimeType', $mimeType) + ->setAttribute('sizeActual', $sizeActual) + ->setAttribute('algorithm', $algorithm) + ->setAttribute('openSSLVersion', $openSSLVersion) + ->setAttribute('openSSLCipher', $openSSLCipher) + ->setAttribute('openSSLTag', $openSSLTag) + ->setAttribute('openSSLIV', $openSSLIV) + ->setAttribute('metadata', $metadata) + ->setAttribute('chunksUploaded', $chunksUploaded); + + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } + } else { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + $file = $file + ->setAttribute('chunksUploaded', $chunksUploaded) + ->setAttribute('metadata', $metadata); + + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + try { + $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket); + + $metadata = null; // was causing leaks as it was passed by reference + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($file, Response::MODEL_FILE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php new file mode 100644 index 0000000000..eccacaafd2 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php @@ -0,0 +1,150 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId') + ->desc('Delete file') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].files.[fileId].delete') + ->label('audits.event', 'file.delete') + ->label('audits.resource', 'file/{request.fileId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'deleteFile', + description: '/docs/references/storage/delete-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_NOCONTENT, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->inject('deviceForFiles') + ->inject('queueForDeletes') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + Response $response, + Database $dbForProject, + Event $queueForEvents, + Device $deviceForFiles, + DeleteEvent $queueForDeletes, + ) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_DELETE); + $valid = $validator->isValid($bucket->getDelete()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + // Read permission should not be required for delete + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + // Make sure we don't delete the file before the document permission check occurs + if ($fileSecurity && !$valid && !$validator->isValid($file->getDelete())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $deviceDeleted = false; + if ($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) { + $deviceDeleted = $deviceForFiles->abort( + $file->getAttribute('path'), + ($file->getAttribute('metadata', [])['uploadId'] ?? '') + ); + } else { + $deviceDeleted = $deviceForFiles->delete($file->getAttribute('path')); + } + + if ($deviceDeleted) { + $queueForDeletes + ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResourceType('bucket/' . $bucket->getId()) + ->setResource('file/' . $fileId) + ; + + try { + if ($fileSecurity && !$valid) { + $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!$deleted) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB'); + } + } else { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to delete file from device'); + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket) + ->setPayload($response->output($file, Response::MODEL_FILE)) + ; + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php new file mode 100644 index 0000000000..45e3b83375 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php @@ -0,0 +1,213 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/download') + ->desc('Get file for download') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFileDownload', + description: '/docs/references/storage/get-file-download.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::ANY, + type: MethodType::LOCATION + )) + ->param('bucketId', '', new UID(), 'Storage bucket ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. + ->param('token', '', new Text(512), 'File token for accessing this file.', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('mode') + ->inject('resourceToken') + ->inject('deviceForFiles') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + ?string $token, + Request $request, + Response $response, + Database $dbForProject, + string $mode, + Document $resourceToken, + Device $deviceForFiles + ) { + /* @type Document $bucket */ + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid && !$isToken) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid && !$isToken) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + /* @type Document $file */ + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { + $end = min(($start + MAX_OUTPUT_CHUNK_SIZE - 1), ($size - 1)); + } + + if ($unit !== 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $response + ->setContentType($file->getAttribute('mimeType')) + ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days + ->addHeader('X-Peak', \memory_get_peak_usage()) + ->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"') + ; + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $deviceForFiles->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + return; + } + $response->send($source); + return; + } + + if (!empty($rangeHeader)) { + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); + return; + } + + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForFiles->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForFiles->read($path)); + } + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php new file mode 100644 index 0000000000..77f163e5fb --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php @@ -0,0 +1,89 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId') + ->desc('Get file') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFile', + description: '/docs/references/storage/get-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + Response $response, + Database $dbForProject, + ) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $response->dynamic($file, Response::MODEL_FILE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php new file mode 100644 index 0000000000..9c4e49d0bb --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php @@ -0,0 +1,287 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/preview') + ->desc('Get file preview') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('cache', true) + ->label('cache.resourceType', 'bucket/{request.bucketId}') + ->label('cache.resource', 'file/{request.fileId}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFilePreview', + description: '/docs/references/storage/get-file-preview.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_NONE + ) + ], + type: MethodType::LOCATION, + contentType: ContentType::IMAGE + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID') + ->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true) + ->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true) + ->param('gravity', Image::GRAVITY_CENTER, new WhiteList(Image::getGravityTypes()), 'Image crop gravity. Can be one of ' . implode(",", Image::getGravityTypes()), true) + ->param('quality', -1, new Range(-1, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to keep existing image quality.', true) + ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) + ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) + ->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true) + ->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) + ->param('rotation', 0, new Range(-360, 360), 'Preview image rotation in degrees. Pass an integer between -360 and 360.', true) + ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) + ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) + // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. + ->param('token', '', new Text(512), 'File token for accessing this file.', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('resourceToken') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->inject('project') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + int $width, + int $height, + string $gravity, + int $quality, + int $borderWidth, + string $borderColor, + int $borderRadius, + float $opacity, + int $rotation, + string $background, + string $output, + ?string $token, + Request $request, + Response $response, + Database $dbForProject, + Document $resourceToken, + Device $deviceForFiles, + Device $deviceForLocal, + Document $project + ) { + + if (!\extension_loaded('imagick')) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); + } + + /* @type Document $bucket */ + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!$bucket->getAttribute('transformations', true) && !$isAPIKey && !$isPrivilegedUser) { + throw new Exception(Exception::STORAGE_BUCKET_TRANSFORMATIONS_DISABLED); + } + + $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid && !$isToken) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid && !$isToken) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + /* @type Document $file */ + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $inputs = Config::getParam('storage-inputs'); + $outputs = Config::getParam('storage-outputs'); + $fileLogos = Config::getParam('storage-logos'); + + $path = $file->getAttribute('path'); + $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); + $algorithm = $file->getAttribute('algorithm', Compression::NONE); + $cipher = $file->getAttribute('openSSLCipher'); + $mime = $file->getAttribute('mimeType'); + if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) System::getEnv('_APP_STORAGE_PREVIEW_LIMIT', APP_STORAGE_READ_BUFFER)) { + if (!\in_array($mime, $inputs)) { + $path = (\array_key_exists($mime, $fileLogos)) ? $fileLogos[$mime] : $fileLogos['default']; + } else { + // it was an image but the file size exceeded the limit + $path = $fileLogos['default_image']; + } + + $algorithm = Compression::NONE; + $cipher = null; + $background = (empty($background)) ? 'eceff1' : $background; + $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); + $deviceForFiles = $deviceForLocal; + } + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + if (empty($output)) { + // when file extension is provided but it's not one of our + // supported outputs we fallback to `jpg` + if (!empty($type) && !array_key_exists($type, $outputs)) { + $type = 'jpg'; + } + + // when file extension is not provided and the mime type is not one of our supported outputs + // we fallback to `jpg` output format + $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; + } + + $startTime = \microtime(true); + + $source = $deviceForFiles->read($path); + + $downloadTime = \microtime(true) - $startTime; + + if (!empty($cipher)) { // Decrypt + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + $decryptionTime = \microtime(true) - $startTime - $downloadTime; + + switch ($algorithm) { + case Compression::ZSTD: + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + $decompressionTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime; + + try { + $image = new Image($source); + } catch (\Exception $e) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, $e->getMessage()); + } + + $image->crop((int) $width, (int) $height, $gravity); + + if (!empty($opacity) || $opacity === 0) { + $image->setOpacity($opacity); + } + + if (!empty($background)) { + $image->setBackground('#' . $background); + } + + if (!empty($borderWidth)) { + $image->setBorder($borderWidth, '#' . $borderColor); + } + + if (!empty($borderRadius)) { + $image->setBorderRadius($borderRadius); + } + + if (!empty($rotation)) { + $image->setRotation(($rotation + 360) % 360); + } + + $data = $image->output($output, $quality); + + $renderingTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime - $decompressionTime; + + $totalTime = \microtime(true) - $startTime; + + Console::info("File preview rendered,project=" . $project->getId() . ",bucket=" . $bucketId . ",file=" . $file->getId() . ",uri=" . $request->getURI() . ",total=" . $totalTime . ",rendering=" . $renderingTime . ",decryption=" . $decryptionTime . ",decompression=" . $decompressionTime . ",download=" . $downloadTime); + + $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; + + //Do not update transformedAt if it's a console user + if (!User::isPrivileged(Authorization::getRoles())) { + $transformedAt = $file->getAttribute('transformedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { + $file->setAttribute('transformedAt', DateTime::now()); + Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + } + } + + $response + ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days + ->setContentType($contentType) + ->file($data); + + unset($image); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php new file mode 100644 index 0000000000..67372435b1 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php @@ -0,0 +1,207 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/push') + ->desc('Get file for push notification') + ->groups(['api', 'storage']) + ->label('scope', 'public') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->param('jwt', '', new Text(2048, 0), 'JSON Web Token to validate', true) + ->inject('response') + ->inject('request') + ->inject('dbForProject') + ->inject('dbForPlatform') + ->inject('project') + ->inject('mode') + ->inject('deviceForFiles') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + string $jwt, + Response $response, + Request $request, + Database $dbForProject, + Database $dbForPlatform, + Document $project, + string $mode, + Device $deviceForFiles + ) { + $decoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); + + try { + $decoded = $decoder->decode($jwt); + } catch (JWTException) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ( + $decoded['projectId'] !== $project->getId() || + $decoded['bucketId'] !== $bucketId || + $decoded['fileId'] !== $fileId + ) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $isInternal = $decoded['internal'] ?? false; + $disposition = $decoded['disposition'] ?? 'inline'; + $dbForProject = $isInternal ? $dbForPlatform : $dbForProject; + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $mimes = Config::getParam('storage-mimes'); + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + $contentType = 'text/plain'; + + if (\in_array($file->getAttribute('mimeType'), $mimes)) { + $contentType = $file->getAttribute('mimeType'); + } + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { + $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); + } + + if ($unit != 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', "bytes $start-$end/$size") + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $response + ->setContentType($contentType) + ->addHeader('Content-Security-Policy', 'script-src none;') + ->addHeader('X-Content-Type-Options', 'nosniff') + ->addHeader('Content-Disposition', $disposition . '; filename="' . $file->getAttribute('name', '') . '"') + ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days + ->addHeader('X-Peak', \memory_get_peak_usage()); + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $deviceForFiles->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + return; + } + $response->send($source); + return; + } + + if (!empty($rangeHeader)) { + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); + return; + } + + $size = $deviceForFiles->getFileSize($path); + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForFiles->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForFiles->read($path)); + } + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php new file mode 100644 index 0000000000..be78cc358b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php @@ -0,0 +1,156 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId') + ->desc('Update file') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].files.[fileId].update') + ->label('audits.event', 'file.update') + ->label('audits.resource', 'file/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'updateFile', + description: '/docs/references/storage/update-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('fileId', '', new UID(), 'File ID.') + ->param('name', null, new Text(128), 'File name.', true) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + ?string $name, + ?array $permissions, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_UPDATE); + $valid = $validator->isValid($bucket->getUpdate()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + // Read permission should not be required for update + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions, [ + Database::PERMISSION_READ, + Database::PERMISSION_UPDATE, + Database::PERMISSION_DELETE, + ]); + + // Users can only manage their own roles, API keys and Admin users can manage any + $roles = Authorization::getRoles(); + if (!User::isApp($roles) && !User::isPrivileged($roles) && !\is_null($permissions)) { + foreach (Database::PERMISSIONS as $type) { + foreach ($permissions as $permission) { + $permission = Permission::parse($permission); + if ($permission->getPermission() != $type) { + continue; + } + $role = (new Role( + $permission->getRole(), + $permission->getIdentifier(), + $permission->getDimension() + ))->toString(); + if (!Authorization::isRole($role)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); + } + } + } + } + + if (\is_null($permissions)) { + $permissions = $file->getPermissions() ?? []; + } + + $file->setAttribute('$permissions', $permissions); + + if (!is_null($name)) { + $file->setAttribute('name', $name); + } + + try { + if ($fileSecurity && !$valid) { + $file = $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file); + } else { + $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket) + ; + + $response->dynamic($file, Response::MODEL_FILE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php new file mode 100644 index 0000000000..41ee95b165 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php @@ -0,0 +1,225 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/view') + ->desc('Get file for view') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFileView', + description: '/docs/references/storage/get-file-view.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::ANY, + type: MethodType::LOCATION + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. + ->param('token', '', new Text(512), 'File token for accessing this file.', true) + ->inject('response') + ->inject('request') + ->inject('dbForProject') + ->inject('mode') + ->inject('resourceToken') + ->inject('deviceForFiles') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + ?string $token, + Response $response, + Request $request, + Database $dbForProject, + string $mode, + Document $resourceToken, + Device $deviceForFiles + ) { + /* @type Document $bucket */ + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid && !$isToken) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid && !$isToken) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + /* @type Document $file */ + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $mimes = Config::getParam('storage-mimes'); + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + $contentType = 'text/plain'; + + if (\in_array($file->getAttribute('mimeType'), $mimes)) { + $contentType = $file->getAttribute('mimeType'); + } + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { + $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); + } + + if ($unit != 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', "bytes $start-$end/$size") + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $response + ->setContentType($contentType) + ->addHeader('Content-Security-Policy', 'script-src none;') + ->addHeader('X-Content-Type-Options', 'nosniff') + ->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"') + ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days + ->addHeader('X-Peak', \memory_get_peak_usage()) + ; + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $deviceForFiles->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + return; + } + $response->send($source); + return; + } + + if (!empty($rangeHeader)) { + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); + return; + } + + $size = $deviceForFiles->getFileSize($path); + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForFiles->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForFiles->read($path)); + } + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php new file mode 100644 index 0000000000..e46fdb2a0a --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php @@ -0,0 +1,155 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files') + ->desc('List files') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'listFiles', + description: '/docs/references/storage/list-files.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE_LIST, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('queries', [], new Files(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Files::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('mode') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + array $queries, + string $search, + bool $includeTotal, + Response $response, + Database $dbForProject, + string $mode + ) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $fileId = $cursor->getValue(); + + if ($fileSecurity && !$valid) { + $cursorDocument = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "File '{$fileId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + + try { + if ($fileSecurity && !$valid) { + $files = $dbForProject->find('bucket_' . $bucket->getSequence(), $queries); + $total = $includeTotal ? $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT) : 0; + } else { + $files = Authorization::skip(fn () => $dbForProject->find('bucket_' . $bucket->getSequence(), $queries)); + $total = $includeTotal ? Authorization::skip(fn () => $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT)) : 0; + } + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } catch (OrderException $e) { + throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + $response->dynamic(new Document([ + 'files' => $files, + 'total' => $total, + ]), Response::MODEL_FILE_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php new file mode 100644 index 0000000000..dd14feef6e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php @@ -0,0 +1,65 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Get bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'getBucket', + description: '/docs/references/storage/get-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + Response $response, + Database $dbForProject + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $response->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php new file mode 100644 index 0000000000..44f0192fb4 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php @@ -0,0 +1,131 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Update bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].update') + ->label('audits.event', 'bucket.update') + ->label('audits.resource', 'bucket/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'updateBucket', + description: '/docs/references/storage/update-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('name', null, new Text(128), 'Bucket name', false) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) + ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) + ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $name, + ?array $permissions, + bool $fileSecurity, + bool $enabled, + ?int $maximumFileSize, + array $allowedFileExtensions, + ?string $compression, + ?bool $encryption, + bool $antivirus, + bool $transformations, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $permissions ??= $bucket->getPermissions(); + $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int) System::getEnv('_APP_STORAGE_LIMIT', 0)); + $allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []); + $enabled ??= $bucket->getAttribute('enabled', true); + $encryption ??= $bucket->getAttribute('encryption', true); + $antivirus ??= $bucket->getAttribute('antivirus', true); + $compression ??= $bucket->getAttribute('compression', Compression::NONE); + $transformations ??= $bucket->getAttribute('transformations', true); + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions); + + $bucket = $dbForProject->updateDocument('buckets', $bucket->getId(), $bucket + ->setAttribute('name', $name) + ->setAttribute('$permissions', $permissions) + ->setAttribute('maximumFileSize', $maximumFileSize) + ->setAttribute('allowedFileExtensions', $allowedFileExtensions) + ->setAttribute('fileSecurity', $fileSecurity) + ->setAttribute('enabled', $enabled) + ->setAttribute('encryption', $encryption) + ->setAttribute('compression', $compression) + ->setAttribute('antivirus', $antivirus) + ->setAttribute('transformations', $transformations)); + + $dbForProject->updateCollection('bucket_' . $bucket->getSequence(), $permissions, $fileSecurity); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()); + + $response->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php new file mode 100644 index 0000000000..74f12852be --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php @@ -0,0 +1,117 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets') + ->desc('List buckets') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'listBuckets', + description: '/docs/references/storage/list-buckets.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET_LIST, + ) + ] + )) + ->param('queries', [], new Buckets(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Buckets::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action( + array $queries, + string $search, + bool $includeTotal, + Response $response, + Database $dbForProject + ) { + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $bucketId = $cursor->getValue(); + $cursorDocument = $dbForProject->getDocument('buckets', $bucketId); + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Bucket '{$bucketId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + try { + $buckets = $dbForProject->find('buckets', $queries); + $total = $includeTotal ? $dbForProject->count('buckets', $filterQueries, APP_LIMIT_COUNT) : 0; + } catch (OrderException $e) { + throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + $response->dynamic(new Document([ + 'buckets' => $buckets, + 'total' => $total, + ]), Response::MODEL_BUCKET_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php new file mode 100644 index 0000000000..b816e83f72 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php @@ -0,0 +1,137 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/:bucketId/usage') + ->desc('Get bucket usage stats') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: null, + name: 'getBucketUsage', + description: '/docs/references/storage/get-bucket-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_USAGE_BUCKETS, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket ID.') + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('project') + ->inject('dbForProject') + ->inject('getLogsDB') + ->callback($this->action(...)); + } + + public function action(string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, callable $getLogsDB) + { + $dbForLogs = call_user_func($getLogsDB, $project); + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES), + str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_STORAGE), + str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED), + ]; + + Authorization::skip(function () use ($dbForProject, $dbForLogs, $bucket, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $db = ($metric === str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED)) + ? $dbForLogs + : $dbForProject; + + $result = $db->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; + $period = $days['period']; + $results = $db->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + + $format = match ($days['period']) { + '1h' => 'Y-m-d\\TH:00:00.000P', + '1d' => 'Y-m-d\\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['data'] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric]['data'][] = [ + 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; + } + } + + $response->dynamic(new Document([ + 'range' => $range, + 'filesTotal' => $usage[$metrics[0]]['total'], + 'filesStorageTotal' => $usage[$metrics[1]]['total'], + 'files' => $usage[$metrics[0]]['data'], + 'storage' => $usage[$metrics[1]]['data'], + 'imageTransformations' => $usage[$metrics[2]]['data'], + 'imageTransformationsTotal' => $usage[$metrics[2]]['total'], + ]), Response::MODEL_USAGE_BUCKETS); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php new file mode 100644 index 0000000000..d29fa7c1b4 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php @@ -0,0 +1,120 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/usage') + ->desc('Get storage usage stats') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: null, + name: 'getUsage', + description: '/docs/references/storage/get-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_USAGE_STORAGE, + ) + ] + )) + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action(string $range, Response $response, Database $dbForProject) + { + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_BUCKETS, + METRIC_FILES, + METRIC_FILES_STORAGE, + ]; + + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $result = $dbForProject->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + $format = match ($days['period']) { + '1h' => 'Y-m-d\\TH:00:00.000P', + '1d' => 'Y-m-d\\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['data'] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric]['data'][] = [ + 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; + } + } + + $response->dynamic(new Document([ + 'range' => $range, + 'bucketsTotal' => $usage[$metrics[0]]['total'], + 'filesTotal' => $usage[$metrics[1]]['total'], + 'filesStorageTotal' => $usage[$metrics[2]]['total'], + 'buckets' => $usage[$metrics[0]]['data'], + 'files' => $usage[$metrics[1]]['data'], + 'storage' => $usage[$metrics[2]]['data'], + ]), Response::MODEL_USAGE_STORAGE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Module.php b/src/Appwrite/Platform/Modules/Storage/Module.php new file mode 100644 index 0000000000..59eb9e87a6 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Module.php @@ -0,0 +1,14 @@ +addService('http', new Http()); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Services/Http.php b/src/Appwrite/Platform/Modules/Storage/Services/Http.php new file mode 100644 index 0000000000..95fe160f8b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Services/Http.php @@ -0,0 +1,51 @@ +type = Service::TYPE_HTTP; + + // Buckets + $this->addAction(CreateBucket::getName(), new CreateBucket()); + $this->addAction(GetBucket::getName(), new GetBucket()); + $this->addAction(ListBuckets::getName(), new ListBuckets()); + $this->addAction(UpdateBucket::getName(), new UpdateBucket()); + $this->addAction(DeleteBucket::getName(), new DeleteBucket()); + + // Files + $this->addAction(CreateFile::getName(), new CreateFile()); + $this->addAction(GetFile::getName(), new GetFile()); + $this->addAction(ListFiles::getName(), new ListFiles()); + $this->addAction(UpdateFile::getName(), new UpdateFile()); + $this->addAction(DeleteFile::getName(), new DeleteFile()); + $this->addAction(GetFilePreview::getName(), new GetFilePreview()); + $this->addAction(GetFileDownload::getName(), new GetFileDownload()); + $this->addAction(GetFileView::getName(), new GetFileView()); + $this->addAction(GetFileForPush::getName(), new GetFileForPush()); + + // Usage + $this->addAction(ListUsage::getName(), new ListUsage()); + $this->addAction(GetBucketUsage::getName(), new GetBucketUsage()); + } +} diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 3ada193cf7..941530d7ed 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; +use Appwrite\Platform\Tasks\Interval; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; @@ -28,6 +29,7 @@ class Tasks extends Service $this ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) + ->addAction(Interval::getName(), new Interval()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) diff --git a/src/Appwrite/Platform/Tasks/Interval.php b/src/Appwrite/Platform/Tasks/Interval.php new file mode 100644 index 0000000000..9d3d782501 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Interval.php @@ -0,0 +1,75 @@ +desc('Schedules tasks on regular intervals by publishing them to our queues') + ->inject('dbForPlatform') + ->inject('queueForCertificates') + ->callback($this->action(...)); + } + + public function action(Database $dbForPlatform, Certificate $queueForCertificates): void + { + Console::title('Interval V1'); + Console::success(APP_NAME . ' interval process v1 has started'); + + $intervalDomainVerification = (int) System::getEnv('_APP_INTERVAL_DOMAIN_VERIFICATION', '60'); // 1 minute + + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalDomainVerification) { + Console::loop(function () use ($dbForPlatform, $queueForCertificates) { + $this->verifyDomain($dbForPlatform, $queueForCertificates); + }, $intervalDomainVerification); + }); + } + + private function verifyDomain(Database $dbForPlatform, Certificate $queueForCertificates): void + { + $time = DatabaseDateTime::now(); + $fromTime = new DateTime('-3 days'); // Max 3 days old + + $rules = $dbForPlatform->find('rules', [ + Query::createdAfter(DatabaseDateTime::format($fromTime)), + Query::equal('status', [RULE_STATUS_CREATED]), // Created but not verified yet + Query::orderAsc('$updatedAt'), // Pick the ones waiting for another attempt for longest + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), // Only current region + Query::limit(30), // Reasonable pagination limit, processable within a minute + ]); + + if (\count($rules) === 0) { + Console::info("[{$time}] No rules for domain verification."); + return; // No rules to verify + } + + Console::info("[{$time}] Found " . \count($rules) . " rules for domain verification, scheduling jobs."); + + foreach ($rules as $rule) { + $queueForCertificates + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_DOMAIN_VERIFICATION) + ->trigger(); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 9c88bc4d4e..c0914c6544 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -125,33 +125,36 @@ class Maintenance extends Action Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]); + if (\count($certificates) === 0) { + Console::info("[{$time}] No certificates for renewal."); + return; + } - if (\count($certificates) > 0) { - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $appRegion = System::getEnv('_APP_REGION', 'default'); - foreach ($certificates as $certificate) { - $domain = $certificate->getAttribute('domain'); - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ + foreach ($certificates as $certificate) { + $domain = $certificate->getAttribute('domain'); + $rule = $isMd5 ? + $dbForPlatform->getDocument('rules', md5($domain)) : + $dbForPlatform->findOne('rules', [ Query::equal('domain', [$domain]), + Query::limit(1) ]); - if ($rule->isEmpty() || $rule->getAttribute('region') !== System::getEnv('_APP_REGION', 'default')) { - continue; - } - - $queueForCertificate - ->setDomain(new Document([ - 'domain' => $certificate->getAttribute('domain') - ])) - ->trigger(); + if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { + continue; } - } else { - Console::info("[{$time}] No certificates for renewal."); + + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); } } diff --git a/src/Appwrite/Platform/Tasks/SDKs.php b/src/Appwrite/Platform/Tasks/SDKs.php index 859e259b7c..d6e3efa559 100644 --- a/src/Appwrite/Platform/Tasks/SDKs.php +++ b/src/Appwrite/Platform/Tasks/SDKs.php @@ -138,7 +138,7 @@ class SDKs extends Action $target = \realpath(__DIR__ . '/../../../../app') . '/sdks/git/' . $language['key'] . '/'; $readme = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/README.md'); $readme = ($readme) ? \file_get_contents($readme) : ''; - $gettingStarted = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); + $gettingStarted = $language['gettingStarted'] ?? \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); $gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : ''; $examples = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md'); $examples = ($examples) ? \file_get_contents($examples) : ''; @@ -381,7 +381,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ->setName($language['name']) ->setNamespace($language['namespace'] ?? 'appwrite') ->setDescription($language['description'] ?? "Appwrite is an open-source backend as a service server that abstracts and simplifies complex and repetitive development tasks behind a very simple to use REST API. Appwrite aims to help you develop your apps faster and in a more secure way. Use the {$language['name']} SDK to integrate your app with the Appwrite server to easily start interacting with all of Appwrite backend APIs and tools. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)") - ->setShortDescription('Appwrite is an open-source self-hosted backend server that abstracts and simplifies complex and repetitive development tasks behind a very simple REST API') + ->setShortDescription($language['shortDescription'] ?? 'Appwrite is an open-source self-hosted backend server that abstracts and simplifies complex and repetitive development tasks behind a very simple REST API') ->setLicense($license) ->setLicenseContent($licenseContent) ->setVersion($language['version']) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 9dc6322163..5132687279 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -3,12 +3,14 @@ namespace Appwrite\Platform\Workers; use Appwrite\Certificates\Adapter as CertificatesAdapter; +use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; -use Appwrite\Network\Validator\DNS; +use Appwrite\Extend\Exception as AppwriteException; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; use Exception; @@ -22,15 +24,12 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; -use Utopia\DNS\Message\Record; +use Utopia\Database\Validator\Authorization as ValidatorAuthorization; use Utopia\Domains\Domain; use Utopia\Locale\Locale; use Utopia\Logger\Log; -use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; -use Utopia\Validator\AnyOf; -use Utopia\Validator\IP; class Certificates extends Action { @@ -42,8 +41,10 @@ class Certificates extends Action /** * @throws Exception */ - public function __construct() + public function __construct(...$params) { + parent::__construct(...$params); + $this ->desc('Certificates worker') ->inject('message') @@ -53,6 +54,7 @@ class Certificates extends Action ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') + ->inject('queueForCertificates') ->inject('log') ->inject('certificates') ->inject('plan') @@ -67,6 +69,7 @@ class Certificates extends Action * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime + * @param Certificate $queueForCertificates * @param Log $log * @param CertificatesAdapter $certificates * @return void @@ -81,6 +84,7 @@ class Certificates extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, + Certificate $queueForCertificates, Log $log, CertificatesAdapter $certificates, array $plan @@ -93,14 +97,96 @@ class Certificates extends Action $document = new Document($payload['domain'] ?? []); $domain = new Domain($document->getAttribute('domain', '')); + $domainType = $document->getAttribute('domainType'); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; $validationDomain = $payload['validationDomain'] ?? null; + $action = $payload['action'] ?? Certificate::ACTION_GENERATION; $log->addTag('domain', $domain->get()); - $domainType = $document->getAttribute('domainType'); + switch ($action) { + case Certificate::ACTION_DOMAIN_VERIFICATION: + $this->handleDomainVerificationAction($domain, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForCertificates, $log, $validationDomain); + break; - $this->execute($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + case Certificate::ACTION_GENERATION: + $this->handleCertificateGenerationAction($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + break; + + default: + throw new Exception('Invalid action: ' . $action); + } + } + + /** + * @param Domain $domain + * @param Database $dbForPlatform + * @param Event $queueForEvents + * @param Webhook $queueForWebhooks + * @param Func $queueForFunctions + * @param Realtime $queueForRealtime + * @param Certificate $queueForCertificates + * @param Log $log + * @param string|null $validationDomain + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + private function handleDomainVerificationAction( + Domain $domain, + Database $dbForPlatform, + Event $queueForEvents, + Webhook $queueForWebhooks, + Func $queueForFunctions, + Realtime $queueForRealtime, + Certificate $queueForCertificates, + Log $log, + ?string $validationDomain = null + ): void { + // Get rule + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); + + // Skip if rule is not desired state (created but not verified yet). + if ($rule->getAttribute('status', '') !== RULE_STATUS_CREATED) { + Console::warning('Domain verification for ' . $rule->getAttribute('domain', '') . ' is not needed.'); + return; + } + + Console::info('Domain verification for ' . $rule->getAttribute('domain', '') . ' started.'); + + try { + // Verify DNS records + $this->validateDomain($rule, $domain, $log, $validationDomain); + // Reset logs and status for the rule + $rule->setAttribute('logs', ''); + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); + + Console::success('Domain verification succeeded.'); + } catch (AppwriteException $err) { + Console::warning('Domain verification failed: ' . $err->getMessage()); + $rule->setAttribute('logs', $err->getMessage()); + } finally { + // Update rule and emit events + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + } + + // Issue a TLS certificate when domain is verified + if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { + $queueForCertificates + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + + Console::success('Certificate generation triggered successfully.'); + } } /** @@ -119,7 +205,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute( + private function handleCertificateGenerationAction( Domain $domain, ?string $domainType, Database $dbForPlatform, @@ -163,26 +249,42 @@ class Certificates extends Action * Note: Renewals are checked and scheduled from maintenance worker */ - // Get current certificate - $certificate = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain->get()])]); + // Get rule document for domain + // TODO: (@Meldiron) Remove after 1.7.x migration + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); - // If we don't have certificate for domain yet, let's create new document. At the end we save it + // Rule not found (or) not in the expected state + if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { + Console::warning('Certificate generation for ' . $domain->get() . ' is skipped as the associated rule is either empty or not in the expected state.'); + return; + } + + // Get associated certificate for the rule + $certificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + // If we don't have certificate for the rule yet, let's create one. if ($certificate->isEmpty()) { $certificate = new Document(); $certificate->setAttribute('domain', $domain->get()); } - $success = false; - try { $date = \date('H:i:s'); $certificate->setAttribute('logs', "\033[90m[{$date}] \033[97mCertificate generation started. \033[0m\n"); + // Persist ASAP so that logs are reset in retry flow and user can see the latest logs on Console. + $certificate = $this->upsertCertificate($rule, $certificate, $dbForPlatform); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { - $mainDomain = $validationDomain ?? $this->getMainDomain(); - $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain, $log); + $this->validateDomain($rule, $domain, $log, $validationDomain); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -191,85 +293,171 @@ class Certificates extends Action } } - // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + // Prepare unique cert name. Using this helps prevent mismatch in configuration when renewing certificates. $certName = ID::unique(); $renewDate = $certificates->issueCertificate($certName, $domain->get(), $domainType); - // Command succeeded, store all data into document - $certificate->setAttribute('logs', 'Certificate successfully generated.'); + // If certificate is generated instantly, we can mark the rule as 'verified'. + if ($certificates->isInstantGeneration($domain->get(), $domainType)) { + $rule->setAttribute('status', RULE_STATUS_VERIFIED); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); + } - // Update certificate info stored in database - $certificate->setAttribute('renewDate', $renewDate); - $certificate->setAttribute('attempts', 0); - $certificate->setAttribute('issueDate', DateTime::now()); - $success = true; + $certificate->setAttributes([ + 'attempts' => 0, // Reset attempts count + 'issueDate' => DateTime::now(), // Store current time as issue date + 'renewDate' => $renewDate, + ]); } catch (Throwable $e) { $logs = $e->getMessage(); $currentLogs = $certificate->getAttribute('logs', ''); $date = \date('H:i:s'); $errorMessage = "\033[90m[{$date}] \033[31mCertificate generation failed: \033[0m\n"; - $certificate->setAttribute('logs', $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000));// Limit to 500kb + $attempts = $certificate->getAttribute('attempts', 0) + 1; // Increase attempts count - // Increase attempts count - $attempts = $certificate->getAttribute('attempts', 0) + 1; - $certificate->setAttribute('attempts', $attempts); + // Update attributes on certificate document + $certificate->setAttributes([ + 'logs' => $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000), // Limit to 500kb + 'attempts' => $attempts, + 'renewDate' => DateTime::now(), // Store current time as renew date to ensure another attempt in next maintenance cycle. + ]); - // Store current time as renew date to ensure another attempt in next maintenance cycle. - $certificate->setAttribute('renewDate', DateTime::now()); + // Mark rule as 'unverified' + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATION_FAILED); // Send email to security email $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails, $plan); throw $e; } finally { - // All actions result in new updatedAt date + // All actions result in new 'updated' date $certificate->setAttribute('updated', DateTime::now()); + // Save certificate document to database + $this->upsertCertificate($rule, $certificate, $dbForPlatform); - // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Update rule and emit events + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } /** - * Save certificate data into database. + * Save certificate data to database. * - * @param string $domain Domain name that certificate is for + * @param Document $rule Rule associated with the domain * @param Document $certificate Certificate document that we need to save - * @param bool $success * @param Database $dbForPlatform Database connection for console - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @return void + * @return Document * @throws \Utopia\Database\Exception * @throws Authorization * @throws Conflict * @throws Structure */ - private function saveCertificateDocument( - string $domain, + private function upsertCertificate( + Document $rule, Document $certificate, - bool $success, + Database $dbForPlatform, + ): Document { + // Decide whether update (or) insert is needed + $existingCertificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + if ($existingCertificate->isEmpty()) { + $certificate->removeAttribute('$sequence'); + $certificate = $dbForPlatform->createDocument('certificates', $certificate); + } else { + $certificate = new Document(\array_merge($existingCertificate->getArrayCopy(), $certificate->getArrayCopy())); + $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); + } + + return $certificate; + } + + /** + * Update all existing domain documents so they have relation to correct certificate document. + * This solves issues: + * - when adding a domain for which there is already a certificate + * - when renew creates new document? It might? + * - overall makes it more reliable + * + * @param Document $rule Rule document that is affected by new certificate + * @param Database $dbForPlatform Database connection for console + * @param Event $queueForEvents Event publisher for events + * @param Webhook $queueForWebhooks Webhook publisher for webhooks + * @param Func $queueForFunctions Function publisher for functions + * @param Realtime $queueForRealtime Realtime publisher for realtime events + * + * @return void + */ + protected function updateRuleAndSendEvents( + Document $rule, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime ): void { - // Check if update or insert required - $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); - if (!$certificateDocument->isEmpty()) { - // Merge new data with current data - $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); - $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); - } else { - $certificate->removeAttribute('$sequence'); - $certificate = $dbForPlatform->createDocument('certificates', $certificate); + $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); + $projectId = $rule->getAttribute('projectId'); + + // Skip events for console project (triggered by auto-ssl generation for 1 click setups) + if ($projectId === 'console') { + return; } - $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + $project = $dbForPlatform->getDocument('projects', $projectId); + if ($project->isEmpty()) { + return; + } + + $ruleModel = new Rule(); + $queueForEvents + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + + /** Trigger Webhook */ + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + + /** Trigger Functions */ + $queueForFunctions + ->from($queueForEvents) + ->trigger(); + + /** Trigger Realtime Events */ + $queueForRealtime + ->setSubscribers(['console', $projectId]) + ->from($queueForEvents) + ->trigger(); + } + + /** + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) + * - Domain must have proper DNS record + * + * @param Document $rule Rule to validate + * @param Domain $domain Domain to validate + * @param Log $log Logger for adding metrics + * @param string|null $validationDomain Override for main domain check + * + * @return void + * @throws Exception + */ + private function validateDomain(Document $rule, Domain $domain, Log $log, ?string $validationDomain = null): void + { + $mainDomain = $validationDomain ?? $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + if (!$isMainDomain) { + $this->verifyRule($rule, $log); + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + } } /** @@ -288,74 +476,7 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported) - * - Domain must have proper DNS record - * - * @param Domain $domain Domain which we validate - * @param bool $isMainDomain In case of master domain, we look for different DNS configurations - * - * @return void - * @throws Exception - */ - private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void - { - if (empty($domain->get())) { - throw new Exception('Missing certificate domain.'); - } - - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain.'); - } - - if (!$isMainDomain) { - $validationStart = \microtime(true); - - $validators = []; - $targetCNAME = new Domain(System::getEnv('_APP_DOMAIN_TARGET_CNAME', '')); - if ($targetCNAME->isKnown() && !$targetCNAME->isTest()) { - $validators[] = new DNS($targetCNAME->get(), Record::TYPE_CNAME); - } - if ((new IP(IP::V4))->isValid(System::getEnv('_APP_DOMAIN_TARGET_A', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_A', ''), Record::TYPE_A); - } - if ((new IP(IP::V6))->isValid(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''), Record::TYPE_AAAA); - } - - // Validate if domain target is properly configured - if (empty($validators)) { - throw new Exception('At least one of domain targets environment variable must be configured.'); - } - - // Verify domain with DNS records - $validator = new AnyOf($validators, AnyOf::TYPE_STRING); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - throw new Exception('Failed to verify domain DNS records.'); - } - - // Ensure CAA won't block certificate issuance - if (!empty(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''))) { - $validationStart = \microtime(true); - $validator = new DNS(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''), Record::TYPE_CAA); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTimingCaa', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - $error = $validator->getDescription(); - $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); - throw new Exception('Failed to verify domain DNS records. CAA records do not allow Appwrite\'s certificate issuer.'); - } - } - } else { - // Main domain validation - // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? - } - } - - /** - * Method to make sure information about error is delivered to admnistrator. + * Method to make sure information about error is delivered to administrator. * * @param string $domain Domain that caused the error * @param string $errorMessage Verbose error message @@ -406,78 +527,4 @@ class Certificates extends Action ->setRecipient(System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))) ->trigger(); } - - /** - * Update all existing domain documents so they have relation to correct certificate document. - * This solved issues: - * - when adding a domain for which there is already a certificate - * - when renew creates new document? It might? - * - overall makes it more reliable - * - * @param string $certificateId ID of a new or updated certificate document - * @param string $domain Domain that is affected by new certificate - * @param bool $success Was certificate generation successful? - * - * @return void - */ - private function updateDomainDocuments( - string $certificateId, - string $domain, - bool $success, - Database $dbForPlatform, - Event $queueForEvents, - Webhook $queueForWebhooks, - Func $queueForFunctions, - Realtime $queueForRealtime - ): void { - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - ]); - - if (!$rule->isEmpty()) { - $rule->setAttribute('certificateId', $certificateId); - $rule->setAttribute('status', $success ? 'verified' : 'unverified'); - $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); - - $projectId = $rule->getAttribute('projectId'); - - // Skip events for console project (triggered by auto-ssl generation for 1 click setups) - if ($projectId === 'console') { - return; - } - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - return; - } - - $ruleModel = new Rule(); - $queueForEvents - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); - - /** Trigger Webhook */ - $queueForWebhooks - ->from($queueForEvents) - ->trigger(); - - /** Trigger Functions */ - $queueForFunctions - ->from($queueForEvents) - ->trigger(); - - /** Trigger Realtime Events */ - $queueForRealtime - ->from($queueForEvents) - ->setSubscribers(['console', $projectId]) - ->trigger(); - } - } } diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index d962ddc8a8..fba5154079 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -122,14 +122,16 @@ class Functions extends Action $log->addTag('type', $type); if (!empty($events)) { - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while ($sum >= $limit) { $functions = $dbForProject->find('functions', [ + Query::select(['$id', 'events']), // Skip variables subqueries + Query::contains('events', $events), Query::limit($limit), Query::offset($offset), - Query::orderAsc('name'), + Query::orderAsc('$sequence'), ]); $sum = \count($functions); @@ -147,6 +149,11 @@ class Functions extends Action continue; } + /** + * get variables subqueries cached + */ + $function = $dbForProject->getDocument('functions', $function->getId()); + Console::success('Iterating function: ' . $function->getAttribute('name')); $this->execute(