diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php new file mode 100644 index 0000000000..a4d28ab487 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php @@ -0,0 +1,167 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/storage/buckets') + ->desc('Create bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].create') + ->label('audits.event', 'bucket.create') + ->label('audits.resource', 'bucket/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'createBucket', + description: '/docs/references/storage/create-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('name', '', new Text(128), 'Bucket name') + ->param('permissions', null, new Nullable(new \Utopia\Database\Validator\Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) + ->param('maximumFileSize', fn(array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn(array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) + ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) + ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $name, + ?array $permissions, + bool $fileSecurity, + bool $enabled, + int $maximumFileSize, + array $allowedFileExtensions, + ?string $compression, + ?bool $encryption, + bool $antivirus, + bool $transformations, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId; + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions) ?? []; + $compression ??= Compression::NONE; + $encryption ??= true; + try { + $files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? []; + if (empty($files)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + 'default' => $attribute['default'] ?? null, + 'format' => $attribute['format'] ?? '' + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForProject->createDocument('buckets', new Document([ + '$id' => $bucketId, + '$collection' => 'buckets', + '$permissions' => $permissions, + 'name' => $name, + 'maximumFileSize' => $maximumFileSize, + 'allowedFileExtensions' => $allowedFileExtensions, + 'fileSecurity' => $fileSecurity, + 'enabled' => $enabled, + 'compression' => $compression, + 'encryption' => $encryption, + 'antivirus' => $antivirus, + 'transformations' => $transformations, + 'search' => implode(' ', [$bucketId, $name]), + ])); + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + $dbForProject->createCollection('bucket_' . $bucket->getSequence(), $attributes, $indexes, permissions: $permissions, documentSecurity: $fileSecurity); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_BUCKET_ALREADY_EXISTS); + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php new file mode 100644 index 0000000000..9523f55e12 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php @@ -0,0 +1,89 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Delete bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('audits.event', 'bucket.delete') + ->label('event', 'buckets.[bucketId].delete') + ->label('audits.resource', 'bucket/{request.bucketId}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'deleteBucket', + description: '/docs/references/storage/delete-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_NOCONTENT, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + Response $response, + Database $dbForProject, + DeleteEvent $queueForDeletes, + Event $queueForEvents + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!$dbForProject->deleteDocument('buckets', $bucketId)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove bucket from DB'); + } + + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($bucket); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setPayload($response->output($bucket, Response::MODEL_BUCKET)) + ; + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php new file mode 100644 index 0000000000..8640dedb8b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -0,0 +1,452 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/storage/buckets/:bucketId/files') + ->desc('Create file') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('audits.event', 'file.create') + ->label('event', 'buckets.[bucketId].files.[fileId].create') + ->label('audits.resource', 'file/{response.$id}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'createFile', + description: '/docs/references/storage/create-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_FILE, + ) + ], + type: MethodType::UPLOAD, + requestType: ContentType::MULTIPART + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new CustomId(), 'File ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('file', [], new File(), 'Binary file. Appwrite SDKs provide helpers to handle file input. [Learn about file input](https://appwrite.io/docs/products/storage/upload-download#input-file).', skipValidation: true) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE])), 'An array of permission strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('user') + ->inject('queueForEvents') + ->inject('mode') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + mixed $file, + ?array $permissions, + Request $request, + Response $response, + Database $dbForProject, + Document $user, + Event $queueForEvents, + string $mode, + Device $deviceForFiles, + Device $deviceForLocal + ) { + $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $allowedPermissions = [ + \Utopia\Database\Database::PERMISSION_READ, + \Utopia\Database\Database::PERMISSION_UPDATE, + \Utopia\Database\Database::PERMISSION_DELETE, + ]; + + // Map aggregate permissions to into the set of individual permissions they represent. + $permissions = Permission::aggregate($permissions, $allowedPermissions); + + // Add permissions for current the user if none were provided. + if (\is_null($permissions)) { + $permissions = []; + if (!empty($user->getId()) && !$isPrivilegedUser) { + foreach ($allowedPermissions as $permission) { + $permissions[] = (new Permission($permission, 'user', $user->getId()))->toString(); + } + } + } + + // Users can only manage their own roles, API keys and Admin users can manage any + $roles = Authorization::getRoles(); + if (!$isAPIKey && !$isPrivilegedUser) { + foreach (\Utopia\Database\Database::PERMISSIONS as $type) { + foreach ($permissions as $permission) { + $permission = Permission::parse($permission); + if ($permission->getPermission() != $type) { + continue; + } + $role = (new Role( + $permission->getRole(), + $permission->getIdentifier(), + $permission->getDimension() + ))->toString(); + if (!Authorization::isRole($role)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); + } + } + } + } + + $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); + if ($maximumFileSize > (int) System::getEnv('_APP_STORAGE_LIMIT', 0)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Maximum bucket file size is larger than _APP_STORAGE_LIMIT'); + } + + $file = $request->getFiles('file'); + + // GraphQL multipart spec adds files with index keys + if (empty($file)) { + $file = $request->getFiles(0); + } + + if (empty($file)) { + throw new Exception(Exception::STORAGE_FILE_EMPTY); + } + + // Make sure we handle a single file and multiple files the same way + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + + $contentRange = $request->getHeader('content-range'); + $fileId = $fileId === 'unique()' ? ID::unique() : $fileId; + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $fileSize = $request->getContentRangeSize(); + $fileId = $request->getHeader('x-appwrite-id', $fileId); + // TODO make `end >= $fileSize` in next breaking version + if (is_null($start) || is_null($end) || is_null($fileSize) || $end > $fileSize) { + throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE); + } + + $idValidator = new UID(); + if (!$idValidator->isValid($fileId)) { + throw new Exception(Exception::STORAGE_INVALID_APPWRITE_ID); + } + + // TODO remove the condition that checks `$end === $fileSize` in next breaking version + if ($end === $fileSize - 1 || $end === $fileSize) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($fileSize / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)) + 1; + } + } + + /** + * Validators + */ + // Check if file type is allowed + $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); + $fileExt = new FileExt($allowedFileExtensions); + if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, 'File extension not allowed'); + } + + // Check if file size is exceeding allowed limit + $fileSizeValidator = new FileSize($maximumFileSize); + if (!$fileSizeValidator->isValid($fileSize)) { + throw new Exception(Exception::STORAGE_INVALID_FILE_SIZE, 'File size not allowed'); + } + + $upload = new Upload(); + if (!$upload->isValid($fileTmpName)) { + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + + // Save to storage + $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); + $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + + $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + $uploaded = $file->getAttribute('chunksUploaded', 0); + $metadata = $file->getAttribute('metadata', []); + + if ($chunk === -1) { + $chunk = $chunks; + } + + if ($uploaded === $chunks) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } + } + + $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + + if (empty($chunksUploaded)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); + } + + if ($chunksUploaded === $chunks) { + if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { + $antivirus = new Network( + System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) + ); + + if (!$antivirus->fileScan($path)) { + $deviceForFiles->delete($path); + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + } + + $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption + $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption + $data = ''; + // Compression + $algorithm = $bucket->getAttribute('compression', Compression::NONE); + if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { + $data = $deviceForFiles->read($path); + switch ($algorithm) { + case Compression::ZSTD: + $compressor = new Zstd(); + break; + case Compression::GZIP: + default: + $compressor = new GZIP(); + break; + } + $data = $compressor->compress($data); + } else { + // reset the algorithm to none as we do not compress the file + // if file size exceedes the APP_STORAGE_READ_BUFFER + // regardless the bucket compression algoorithm + $algorithm = Compression::NONE; + } + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + if (empty($data)) { + $data = $deviceForFiles->read($path); + } + $key = System::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if (!empty($data)) { + if (!$deviceForFiles->write($path, $data, $mimeType)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); + } + } + + $sizeActual = $deviceForFiles->getFileSize($path); + + $openSSLVersion = null; + $openSSLCipher = null; + $openSSLTag = null; + $openSSLIV = null; + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $fileSize, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + $file = $file + ->setAttribute('$permissions', $permissions) + ->setAttribute('signature', $fileHash) + ->setAttribute('mimeType', $mimeType) + ->setAttribute('sizeActual', $sizeActual) + ->setAttribute('algorithm', $algorithm) + ->setAttribute('openSSLVersion', $openSSLVersion) + ->setAttribute('openSSLCipher', $openSSLCipher) + ->setAttribute('openSSLTag', $openSSLTag) + ->setAttribute('openSSLIV', $openSSLIV) + ->setAttribute('metadata', $metadata) + ->setAttribute('chunksUploaded', $chunksUploaded); + + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } + } else { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + $file = $file + ->setAttribute('chunksUploaded', $chunksUploaded) + ->setAttribute('metadata', $metadata); + + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + try { + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket); + + $metadata = null; // was causing leaks as it was passed by reference + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($file, Response::MODEL_FILE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php new file mode 100644 index 0000000000..09e019335a --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php @@ -0,0 +1,17 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId') + ->desc('Get file') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFile', + description: '/docs/references/storage/get-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('mode') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + Response $response, + Database $dbForProject, + string $mode + ) { + $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $response->dynamic($file, Response::MODEL_FILE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php new file mode 100644 index 0000000000..5f33f9f323 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php @@ -0,0 +1,17 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files') + ->desc('List files') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'listFiles', + description: '/docs/references/storage/list-files.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE_LIST, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('queries', [], new Files(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Files::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('mode') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + array $queries, + string $search, + bool $includeTotal, + Response $response, + Database $dbForProject, + string $mode + ) { + $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $queries = Query::parseQueries($queries); + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $fileId = $cursor->getValue(); + + if ($fileSecurity && !$valid) { + $cursorDocument = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $cursorDocument = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "File '{$fileId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + + try { + if ($fileSecurity && !$valid) { + $files = $dbForProject->find('bucket_' . $bucket->getSequence(), $queries); + $total = $includeTotal ? $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT) : 0; + } else { + $files = Authorization::skip(fn() => $dbForProject->find('bucket_' . $bucket->getSequence(), $queries)); + $total = $includeTotal ? Authorization::skip(fn() => $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT)) : 0; + } + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } catch (OrderException $e) { + throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + $response->dynamic(new Document([ + 'files' => $files, + 'total' => $total, + ]), Response::MODEL_FILE_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php new file mode 100644 index 0000000000..dd14feef6e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php @@ -0,0 +1,65 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Get bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'getBucket', + description: '/docs/references/storage/get-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + Response $response, + Database $dbForProject + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $response->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php new file mode 100644 index 0000000000..9f83479671 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php @@ -0,0 +1,131 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Update bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].update') + ->label('audits.event', 'bucket.update') + ->label('audits.resource', 'bucket/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'updateBucket', + description: '/docs/references/storage/update-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('name', null, new Text(128), 'Bucket name', false) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) + ->param('maximumFileSize', fn(array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn(array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) + ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) + ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $name, + ?array $permissions, + bool $fileSecurity, + bool $enabled, + ?int $maximumFileSize, + array $allowedFileExtensions, + ?string $compression, + ?bool $encryption, + bool $antivirus, + bool $transformations, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $permissions ??= $bucket->getPermissions(); + $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int) System::getEnv('_APP_STORAGE_LIMIT', 0)); + $allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []); + $enabled ??= $bucket->getAttribute('enabled', true); + $encryption ??= $bucket->getAttribute('encryption', true); + $antivirus ??= $bucket->getAttribute('antivirus', true); + $compression ??= $bucket->getAttribute('compression', Compression::NONE); + $transformations ??= $bucket->getAttribute('transformations', true); + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions); + + $bucket = $dbForProject->updateDocument('buckets', $bucket->getId(), $bucket + ->setAttribute('name', $name) + ->setAttribute('$permissions', $permissions) + ->setAttribute('maximumFileSize', $maximumFileSize) + ->setAttribute('allowedFileExtensions', $allowedFileExtensions) + ->setAttribute('fileSecurity', $fileSecurity) + ->setAttribute('enabled', $enabled) + ->setAttribute('encryption', $encryption) + ->setAttribute('compression', $compression) + ->setAttribute('antivirus', $antivirus) + ->setAttribute('transformations', $transformations)); + + $dbForProject->updateCollection('bucket_' . $bucket->getSequence(), $permissions, $fileSecurity); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()); + + $response->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php new file mode 100644 index 0000000000..74f12852be --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php @@ -0,0 +1,117 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets') + ->desc('List buckets') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'listBuckets', + description: '/docs/references/storage/list-buckets.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET_LIST, + ) + ] + )) + ->param('queries', [], new Buckets(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Buckets::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action( + array $queries, + string $search, + bool $includeTotal, + Response $response, + Database $dbForProject + ) { + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $bucketId = $cursor->getValue(); + $cursorDocument = $dbForProject->getDocument('buckets', $bucketId); + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Bucket '{$bucketId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + try { + $buckets = $dbForProject->find('buckets', $queries); + $total = $includeTotal ? $dbForProject->count('buckets', $filterQueries, APP_LIMIT_COUNT) : 0; + } catch (OrderException $e) { + throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + $response->dynamic(new Document([ + 'buckets' => $buckets, + 'total' => $total, + ]), Response::MODEL_BUCKET_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php new file mode 100644 index 0000000000..496ac54582 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php @@ -0,0 +1,17 @@ +addService('http', new Http()); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Services/Http.php b/src/Appwrite/Platform/Modules/Storage/Services/Http.php new file mode 100644 index 0000000000..e60571eff0 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Services/Http.php @@ -0,0 +1,51 @@ +type = Service::TYPE_HTTP; + + // Buckets + $this->addAction(CreateBucket::getName(), new CreateBucket()); + $this->addAction(GetBucket::getName(), new GetBucket()); + $this->addAction(ListBuckets::getName(), new ListBuckets()); + $this->addAction(UpdateBucket::getName(), new UpdateBucket()); + $this->addAction(DeleteBucket::getName(), new DeleteBucket()); + + // Files + $this->addAction(CreateFile::getName(), new CreateFile()); + $this->addAction(GetFile::getName(), new GetFile()); + $this->addAction(ListFiles::getName(), new ListFiles()); + $this->addAction(UpdateFile::getName(), new UpdateFile()); + $this->addAction(DeleteFile::getName(), new DeleteFile()); + $this->addAction(GetFilePreview::getName(), new GetFilePreview()); + $this->addAction(GetFileDownload::getName(), new GetFileDownload()); + $this->addAction(GetFileView::getName(), new GetFileView()); + $this->addAction(GetFileForPush::getName(), new GetFileForPush()); + + // Usage + $this->addAction(ListUsage::getName(), new ListUsage()); + $this->addAction(GetBucketUsage::getName(), new GetBucketUsage()); + } +}