diff --git a/app/config/collections.php b/app/config/collections.php index 445e48914a..2789501cf2 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -6,6 +6,15 @@ use Utopia\Database\Database; $providers = Config::getParam('providers', []); $auth = Config::getParam('auth', []); +/** + * $collection => id of the parent collection where this will be inserted + * $id => id of this collection + * name => name of this collection + * project => whether or not this collection should be created per project + * attributes => list of attributes + * indexes => list of indexes + */ + $collections = [ 'collections' => [ '$collection' => Database::METADATA, @@ -2045,9 +2054,7 @@ $collections = [ 'buckets' => [ '$collection' => Database::METADATA, '$id' => 'buckets', - '$permissions' => ['read' => ['*']], 'name' => 'Buckets', - 'structure' => true, 'attributes' => [ [ '$id' => 'dateCreated', @@ -2064,7 +2071,7 @@ $collections = [ 'type' => Database::VAR_INTEGER, 'size' => 0, 'format' => '', - 'signed' => true, + 'signed' => false, 'required' => false, 'array' => false, 'filters' => [], @@ -2123,8 +2130,8 @@ $collections = [ [ '$id' => 'maximumFileSize', 'type' => Database::VAR_INTEGER, - 'signed' => true, - 'size' => 0, + 'signed' => false, + 'size' => 8, 'format' => '', 'filters' => [], 'required' => true, @@ -2324,6 +2331,228 @@ $collections = [ ], ] ], + 'files' => [ + '$collection' => 'buckets', + '$id' => 'files', + '$name' => 'Files', + 'attributes' => [ + [ + '$id' => 'dateCreated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + 'array' => false, + '$id' => 'bucketId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'path', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'signature', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'mimeType', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'metadata', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['json'], + ], + [ + '$id' => 'sizeOriginal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 8, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'sizeActual', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 8, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'algorithm', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'comment', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLVersion', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 64, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLCipher', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 64, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLTag', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'openSSLIV', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_bucket', + 'type' => Database::INDEX_KEY, + 'attributes' => ['bucketId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ] + ], ]; return $collections; diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 9149bbff98..19d0288e36 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -452,11 +452,15 @@ App::post('/v1/functions/:functionId/tags') ->inject('response') ->inject('dbForProject') ->inject('usage') - ->action(function ($functionId, $command, $file, $request, $response, $dbForProject, $usage) { - /** @var Appwrite\Utopia\Request $request */ + ->inject('deviceFunctions') + ->inject('deviceLocal') + ->action(function ($functionId, $command, $file, $request, $response, $dbForProject, $usage, $deviceFunctions, $deviceLocal) { + /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $usage */ + /** @var Utopia\Storage\Device $deviceFunctions */ + /** @var Utopia\Storage\Device $deviceLocal */ $function = $dbForProject->getDocument('functions', $functionId); @@ -465,7 +469,6 @@ App::post('/v1/functions/:functionId/tags') } $file = $request->getFiles('code'); - $device = Storage::getDevice('functions'); $fileExt = new FileExt([FileExt::TYPE_GZIP]); $fileSizeValidator = new FileSize(App::getEnv('_APP_STORAGE_LIMIT', 0)); $upload = new Upload(); @@ -516,8 +519,8 @@ App::post('/v1/functions/:functionId/tags') } // Save to storage - $fileSize ??= Storage::getDevice('self')->getFileSize($fileTmpName); - $path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION)); + $fileSize ??= $deviceLocal->getFileSize($fileTmpName); + $path = $deviceFunctions->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION)); $tag = $dbForProject->getDocument('tags', $tagId); @@ -528,14 +531,14 @@ App::post('/v1/functions/:functionId/tags') } } - $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks); + $chunksUploaded = $deviceFunctions->upload($fileTmpName, $path, $chunk, $chunks); if (empty($chunksUploaded)) { throw new Exception('Failed moving file', 500); } if($chunksUploaded == $chunks) { - $fileSize = $device->getFileSize($path); + $fileSize = $deviceFunctions->getFileSize($path); if ($tag->isEmpty()) { $tag = $dbForProject->createDocument('tags', new Document([ @@ -689,10 +692,12 @@ App::delete('/v1/functions/:functionId/tags/:tagId') ->inject('response') ->inject('dbForProject') ->inject('usage') - ->action(function ($functionId, $tagId, $response, $dbForProject, $usage) { + ->inject('deviceFunctions') + ->action(function ($functionId, $tagId, $response, $dbForProject, $usage, $deviceFunctions) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $usage */ + /** @var Utopia\Storage\Device $deviceFunctions */ $function = $dbForProject->getDocument('functions', $functionId); @@ -710,9 +715,7 @@ App::delete('/v1/functions/:functionId/tags/:tagId') throw new Exception('Tag not found', 404); } - $device = Storage::getDevice('functions'); - - if ($device->delete($tag->getAttribute('path', ''))) { + if ($deviceFunctions->delete($tag->getAttribute('path', ''))) { if (!$dbForProject->deleteDocument('tags', $tag->getId())) { throw new Exception('Failed to remove tag from DB', 500); } diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 9c4f9bc9bf..5418856cc3 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -114,6 +114,9 @@ App::post('/v1/projects') $adapter->setup(); foreach ($collections as $key => $collection) { + if(($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } $attributes = []; $indexes = []; diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8003e25438..932d135991 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -69,222 +69,37 @@ App::post('/v1/storage/buckets') $bucketId = $bucketId === 'unique()' ? $dbForProject->getId() : $bucketId; try { - $dbForProject->createCollection('bucket_' . $bucketId, [ - new Document([ - '$id' => 'dateCreated', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - 'array' => false, - '$id' => 'bucketId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'name', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'path', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'signature', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'mimeType', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'metadata', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['json'], - ]), - new Document([ - '$id' => 'sizeOriginal', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'sizeActual', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'algorithm', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 255, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'comment', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLVersion', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 64, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLCipher', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 64, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLTag', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'openSSLIV', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'chunksTotal', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'chunksUploaded', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - new Document([ - '$id' => 'search', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ]), - ], [ - new Document([ - '$id' => '_key_search', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ]), - new Document([ - '$id' => '_key_bucket', - 'type' => Database::INDEX_KEY, - 'attributes' => ['bucketId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ]), - ]); + $files = Config::getParam('collections', [])['files'] ?? []; + if(empty($files)) { + throw new Exception('Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForProject->createCollection('bucket_' . $bucketId, $attributes, $indexes); $bucket = $dbForProject->createDocument('buckets', new Document([ '$id' => $bucketId, @@ -543,13 +358,17 @@ App::post('/v1/storage/buckets/:bucketId/files') ->inject('audits') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $file, $read, $write, $request, $response, $dbForProject, $user, $audits, $usage, $mode) { + ->inject('deviceFiles') + ->inject('deviceLocal') + ->action(function ($bucketId, $fileId, $file, $read, $write, $request, $response, $dbForProject, $user, $audits, $usage, $mode, $deviceFiles, $deviceLocal) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ + /** @var Utopia\Storage\Device $deviceLocal */ /** @var string $mode */ $bucket = $dbForProject->getDocument('buckets', $bucketId); @@ -600,9 +419,7 @@ App::post('/v1/storage/buckets/:bucketId/files') throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500); } - $fileSizeValidator = new FileSize($maximumFileSize); - $upload = new Upload(); - + $file = $request->getFiles('file'); if (empty($file)) { throw new Exception('No file sent', 400); } @@ -627,7 +444,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } if ($end === $fileSize) { - //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk $chunks = $chunk = -1; } else { // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) @@ -636,26 +453,31 @@ App::post('/v1/storage/buckets/:bucketId/files') } } - // Check if file type is allowed (feature for project settings?) + /** + * Validators + */ + // Check if file type is allowed + $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); + $fileExt = new FileExt($allowedFileExtensions); if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { throw new Exception('File extension not allowed', 400); } - if (!$fileSizeValidator->isValid($fileSize)) { // Check if file size is exceeding allowed limit + // Check if file size is exceeding allowed limit + $fileSizeValidator = new FileSize($maximumFileSize); + if (!$fileSizeValidator->isValid($fileSize)) { throw new Exception('File size not allowed', 400); } - - $device = Storage::getDevice('files'); - $localDevice = Storage::getDevice('self'); - + + $upload = new Upload(); if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $fileSize ??= $localDevice->getFileSize($fileTmpName); - $path = $device->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); - $path = str_ireplace($device->getRoot(), $device->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + $fileSize ??= $deviceLocal->getFileSize($fileTmpName); + $path = $deviceFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($deviceFiles->getRoot(), $deviceFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root if($permissionBucket) { $file = Authorization::skip(function() use ($dbForProject, $bucketId, $fileId) { @@ -665,7 +487,7 @@ App::post('/v1/storage/buckets/:bucketId/files') $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); } - $metadata = ['content_type' => $localDevice->getFileMimeType($fileTmpName)]; + $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $metadata = $file->getAttribute('metadata', []); @@ -674,7 +496,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } } - $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + $chunksUploaded = $deviceFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); if (empty($chunksUploaded)) { throw new Exception('Failed uploading file', 500); } @@ -687,23 +509,23 @@ App::post('/v1/storage/buckets/:bucketId/files') (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)); if (!$antiVirus->fileScan($path)) { - $device->delete($path); - throw new Exception('Invalid file', 403); + $deviceFiles->delete($path); + throw new Exception('Invalid file', 400); } } - $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + $mimeType = $deviceFiles->getFileMimeType($path); // Get mime-type before compression and encryption $data = ''; // Compression if ($fileSize <= APP_STORAGE_READ_BUFFER) { - $data = $device->read($path); + $data = $deviceFiles->read($path); $compressor = new GZIP(); $data = $compressor->compress($data); } if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { if(empty($data)) { - $data = $device->read($path); + $data = $deviceFiles->read($path); } $key = App::getEnv('_APP_OPENSSL_KEY_V1'); $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); @@ -711,15 +533,15 @@ App::post('/v1/storage/buckets/:bucketId/files') } if (!empty($data)) { - if (!$device->write($path, $data, $mimeType)) { + if (!$deviceFiles->write($path, $data, $mimeType)) { throw new Exception('Failed to save file', 500); } } - $sizeActual = $device->getFileSize($path); + $sizeActual = $deviceFiles->getFileSize($path); $algorithm = empty($compressor) ? '' : $compressor->getName(); - $fileHash = $device->getFileHash($path); + $fileHash = $deviceFiles->getFileHash($path); if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { $openSSLVersion = '1'; @@ -1036,23 +858,20 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('dbForProject') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForProject, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForProject, $usage, $mode, $deviceFiles) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ - $storage = 'files'; - if (!\extension_loaded('imagick')) { throw new Exception('Imagick extension is missing', 500); } - if (!Storage::exists($storage)) { - throw new Exception('No such storage device', 400); - } $bucket = $dbForProject->getDocument('buckets', $bucketId); if ($bucket->isEmpty() @@ -1077,7 +896,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $fileLogos = Config::getParam('storage-logos'); $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT'; // 45 days cache - $key = \md5($fileId.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output); + $key = \md5($fileId.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$output); if ($bucket->getAttribute('permission')==='bucket') { // skip authorization @@ -1104,13 +923,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $cipher = null; $background = (empty($background)) ? 'eceff1' : $background; $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $key = \md5($path.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output); + $key = \md5($path.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$output); } $compressor = new GZIP(); - $device = Storage::getDevice('files'); - if (!$device->exists($path)) { + if (!$deviceFiles->exists($path)) { throw new Exception('File not found', 404); } @@ -1128,7 +946,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ; } - $source = $device->read($path); + $source = $deviceFiles->read($path); if (!empty($cipher)) { // Decrypt $source = OpenSSL::decrypt( @@ -1209,11 +1027,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->inject('dbForProject') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $request, $response, $dbForProject, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $request, $response, $dbForProject, $usage, $mode, $deviceFiles) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ $bucket = $dbForProject->getDocument('buckets', $bucketId); @@ -1244,10 +1064,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') } $path = $file->getAttribute('path', ''); - - $device = Storage::getDevice('files'); - if (!$device->exists($path)) { + if (!$deviceFiles->exists($path)) { throw new Exception('File not found in ' . $path, 404); } @@ -1288,7 +1106,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $device->read($path); + $source = $deviceFiles->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), @@ -1301,7 +1119,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') if (!empty($file->getAttribute('algorithm', ''))) { if (empty($source)) { - $source = $device->read($path); + $source = $deviceFiles->read($path); } $compressor = new GZIP(); $source = $compressor->decompress($source); @@ -1315,14 +1133,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') } if (!empty($rangeHeader)) { - $response->send($device->read($path, $start, ($end - $start + 1))); + $response->send($deviceFiles->read($path, $start, ($end - $start + 1))); } if ($size > APP_STORAGE_READ_BUFFER) { - $response->addHeader('Content-Length', $device->getFileSize($path)); + $response->addHeader('Content-Length', $deviceFiles->getFileSize($path)); for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $device->read( + $deviceFiles->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -1331,7 +1149,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ); } } else { - $response->send($device->read($path)); + $response->send($deviceFiles->read($path)); } }); @@ -1354,11 +1172,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->inject('dbForProject') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $response, $request, $dbForProject, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $response, $request, $dbForProject, $usage, $mode, $deviceFiles) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Swoole\Request $request */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ $bucket = $dbForProject->getDocument('buckets', $bucketId); @@ -1392,8 +1212,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $path = $file->getAttribute('path', ''); - $device = Storage::getDevice('files'); - if (!$device->exists($path)) { + if (!$deviceFiles->exists($path)) { throw new Exception('File not found in ' . $path, 404); } @@ -1439,7 +1258,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $device->read($path); + $source = $deviceFiles->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), @@ -1452,7 +1271,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') if (!empty($file->getAttribute('algorithm', ''))) { if (empty($source)) { - $source = $device->read($path); + $source = $deviceFiles->read($path); } $compressor = new GZIP(); $source = $compressor->decompress($source); @@ -1471,15 +1290,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') } if (!empty($rangeHeader)) { - $response->send($device->read($path, $start, ($end - $start + 1))); + $response->send($deviceFiles->read($path, $start, ($end - $start + 1))); } - $size = $device->getFileSize($path); + $size = $deviceFiles->getFileSize($path); if ($size > APP_STORAGE_READ_BUFFER) { - $response->addHeader('Content-Length', $device->getFileSize($path)); + $response->addHeader('Content-Length', $deviceFiles->getFileSize($path)); for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $device->read( + $deviceFiles->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -1488,7 +1307,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ); } } else { - $response->send($device->read($path)); + $response->send($deviceFiles->read($path)); } }); @@ -1615,13 +1434,15 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->inject('audits') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $response, $dbForProject, $events, $audits, $usage, $mode) { + ->inject('deviceFiles') + ->action(function ($bucketId, $fileId, $response, $dbForProject, $events, $audits, $usage, $mode, $deviceFiles) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ + /** @var Utopia\Storage\Device $deviceFiles */ /** @var string $mode */ $bucket = $dbForProject->getDocument('buckets', $bucketId); @@ -1651,13 +1472,11 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') throw new Exception('File not found', 404); } - $device = Storage::getDevice('files'); - $deviceDeleted = false; if($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) { - $deviceDeleted = $device->abort($file->getAttribute('path')); + $deviceDeleted = $deviceFiles->abort($file->getAttribute('path')); } else { - $deviceDeleted = $device->delete($file->getAttribute('path')); + $deviceDeleted = $deviceFiles->delete($file->getAttribute('path')); } if ($deviceDeleted) { diff --git a/app/http.php b/app/http.php index 4414ddf728..cc71dd9a98 100644 --- a/app/http.php +++ b/app/http.php @@ -13,6 +13,7 @@ use Utopia\Config\Config; use Utopia\Database\Validator\Authorization; use Utopia\Audit\Audit; use Utopia\Abuse\Adapters\TimeLimit; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Swoole\Files; use Appwrite\Utopia\Request; @@ -109,10 +110,12 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { } foreach ($collections as $key => $collection) { + if(($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } if(!$dbForConsole->getCollection($key)->isEmpty()) { continue; } - Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); $attributes = []; @@ -141,8 +144,62 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { } $dbForConsole->createCollection($key, $attributes, $indexes); - } + + if($dbForConsole->getDocument('buckets', 'default')->isEmpty()) { + Console::success('[Setup] - Creating default bucket...'); + $dbForConsole->createDocument('buckets', new Document([ + '$id' => 'default', + '$collection' => 'buckets', + 'dateCreated' => \time(), + 'dateUpdated' => \time(), + 'name' => 'Default', + 'permission' => 'file', + 'maximumFileSize' => (int) App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB + 'allowedFileExtensions' => [], + 'enabled' => true, + 'adapter' => '', + 'encryption' => true, + 'antiVirus' => true, + '$read' => ['role:all'], + '$write' => ['role:all'], + 'search' => 'buckets Default', + ])); + + Console::success('[Setup] - Creating files collection for default bucket...'); + $files = $collections['files'] ?? []; + if(empty($files)) { + throw new Exception('Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForConsole->createCollection('bucket_' . 'default', $attributes, $indexes); + } + Console::success('[Setup] - Server database init completed...'); }); diff --git a/app/init.php b/app/init.php index 6cfd02b9d8..293013c7a5 100644 --- a/app/init.php +++ b/app/init.php @@ -50,6 +50,10 @@ use Swoole\Database\PDOPool; use Swoole\Database\RedisConfig; use Swoole\Database\RedisPool; use Utopia\Database\Query; +use Utopia\Storage\Storage; +use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\S3; +use Utopia\Storage\Device\DoSpaces; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -834,6 +838,53 @@ App::setResource('dbForConsole', function($db, $cache) { return $database; }, ['db', 'cache']); + +App::setResource('deviceLocal', function() { + return new Local(); +}); + +App::setResource('deviceFiles', function($project) { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DoSpaces(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + } +}, ['project']); + +App::setResource('deviceFunctions', function($project) { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DoSpaces(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + } +}, ['project']); + App::setResource('mode', function($request) { /** @var Appwrite\Utopia\Request $request */ diff --git a/public/scripts/views/forms/upload.js b/public/scripts/views/forms/upload.js index 9fdd1bda66..472176eb28 100644 --- a/public/scripts/views/forms/upload.js +++ b/public/scripts/views/forms/upload.js @@ -108,7 +108,7 @@ expression.parse(element.dataset["write"] || "[]") ); - sdk.storage.createFile('unique()', files[0], read, write, 1).then( + sdk.storage.createFile('default', 'unique()', files[0], read, write, 1).then( function(response) { onComplete(message);