diff --git a/app/config/collections.php b/app/config/collections.php index c39e226a23..988ae39f0b 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1769,6 +1769,28 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'search', 'type' => Database::VAR_STRING, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 8488b341f0..4cb38f2f0d 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -475,42 +475,102 @@ App::post('/v1/functions/:functionId/tags') } // Make sure we handle a single file and multiple files the same way - $file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed throw new Exception('File type not allowed', 400); } - if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit + $contentRange = $request->getHeader('content-range'); + $tagId = $dbForProject->getId(); + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $size = $request->getContentRangeSize(); + $tagId = $request->getHeader('x-appwrite-id', $tagId); + if(is_null($start) || is_null($end) || is_null($size)) { + throw new Exception('Invalid content-range header', 400); + } + + if ($end == $size) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($size / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)) + 1; + } + } + + if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit throw new Exception('File size not allowed', 400); } - if (!$upload->isValid($file['tmp_name'])) { + if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $size = $device->getFileSize($file['tmp_name']); - $path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION)); + $size ??= $device->getFileSize($fileTmpName); + $path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION)); - if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move' + $tag = $dbForProject->getDocument('tags', $tagId); + + if(!$tag->isEmpty()) { + $chunks = $tag->getAttribute('chunksTotal', 1); + if($chunk == -1) { + $chunk = $chunks; + } + } + + $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks); + + if (empty($chunksUploaded)) { throw new Exception('Failed moving file', 500); } - $tagId = $dbForProject->getId(); - $tag = $dbForProject->createDocument('tags', new Document([ - '$id' => $tagId, - '$read' => [], - '$write' => [], - 'functionId' => $function->getId(), - 'dateCreated' => time(), - 'command' => $command, - 'path' => $path, - 'size' => $size, - 'search' => implode(' ', [$tagId, $command]), - ])); + if($chunksUploaded == $chunks) { + $size = $device->getFileSize($path); + + if ($tag->isEmpty()) { + $tag = $dbForProject->createDocument('tags', new Document([ + '$id' => $tagId, + '$read' => [], + '$write' => [], + 'functionId' => $function->getId(), + 'dateCreated' => time(), + 'command' => $command, + 'path' => $path, + 'size' => $size, + 'search' => implode(' ', [$tagId, $command]), + ])); + } else { + $tag = $dbForProject->updateDocument('tags', $tagId, $tag->setAttribute('size', $size)); + } + } else { + if($tag->isEmpty()) { + $tag = $dbForProject->createDocument('tags', new Document([ + '$id' => $tagId, + '$read' => [], + '$write' => [], + 'functionId' => $function->getId(), + 'dateCreated' => time(), + 'command' => $command, + 'path' => $path, + 'size' => 0, + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$tagId, $command]), + ])); + } else { + $tag = $dbForProject->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded)); + } + } $usage ->setParam('storage', $tag->getAttribute('size', 0)) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 619bedb751..ed3cd59343 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -2,19 +2,13 @@ use Appwrite\Auth\Auth; use Utopia\App; -use Utopia\Exception; -use Utopia\Validator\ArrayList; -use Utopia\Validator\WhiteList; -use Utopia\Validator\Range; -use Utopia\Validator\Text; -use Utopia\Validator\Boolean; -use Utopia\Validator\HexColor; -use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Filesystem; use Appwrite\ClamAV\Network; use Utopia\Database\Validator\Authorization; use Appwrite\Database\Validator\CustomId; use Utopia\Database\Document; +use Utopia\Database\Query; +use Utopia\Exception; use Utopia\Database\Validator\UID; use Utopia\Storage\Storage; use Utopia\Storage\Validator\File; @@ -24,11 +18,17 @@ use Utopia\Storage\Compression\Algorithms\GZIP; use Utopia\Image\Image; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Response; +use Utopia\Cache\Cache; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Exception\Duplicate; +use Utopia\Validator\ArrayList; +use Utopia\Validator\Boolean; +use Utopia\Validator\HexColor; use Utopia\Validator\Integer; -use Utopia\Database\Query; +use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; +use Utopia\Validator\Range; use Utopia\Database\Validator\Permissions; use Utopia\Storage\Validator\FileExt; use Utopia\Database\Exception\Duplicate as DuplicateException; @@ -55,7 +55,7 @@ App::post('/v1/storage/buckets') ->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true) ->param('enabled', true, new Boolean(), 'Is bucket enabled?', true) ->param('adapter', 'local', new WhiteList(['local']), 'Storage adapter.', true) - ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true) ->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') ->inject('dbForProject') @@ -225,6 +225,28 @@ App::post('/v1/storage/buckets') 'array' => false, 'filters' => [], ]), + new Document([ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ]), + new Document([ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ]), new Document([ '$id' => 'search', 'type' => Database::VAR_STRING, @@ -237,13 +259,6 @@ App::post('/v1/storage/buckets') 'filters' => [], ]), ], [ - new Document([ - '$id' => '_key_bucket', - 'type' => Database::INDEX_KEY, - 'attributes' => ['bucketId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ]), new Document([ '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, @@ -251,6 +266,13 @@ App::post('/v1/storage/buckets') 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ]), + new Document([ + '$id' => '_key_bucket', + 'type' => Database::INDEX_KEY, + 'attributes' => ['bucketId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ]), ]); $bucket = $dbForProject->createDocument('buckets', new Document([ @@ -379,7 +401,7 @@ App::put('/v1/storage/buckets/:bucketId') ->param('maximumFileSize', null, new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self hosted version you can change the limit by changing _APP_STORAGE_LIMIT environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true) ->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true) ->param('enabled', true, new Boolean(), 'Is bucket enabled?', true) - ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true) ->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') ->inject('dbForProject') @@ -398,7 +420,7 @@ App::put('/v1/storage/buckets/:bucketId') } $read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions - $write ??= $bucket->getAttribute('$write',[]); // By default inherit write permissions + $write ??= $bucket->getAttribute('$write', []); // By default inherit write permissions $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int)App::getEnv('_APP_STORAGE_LIMIT', 0)); $allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []); $enabled ??= $bucket->getAttribute('enabled', true); @@ -484,7 +506,7 @@ App::delete('/v1/storage/buckets/:bucketId') }); App::post('/v1/storage/buckets/:bucketId/files') - ->alias('/v1/storage/files',['bucketId' => 'default']) + ->alias('/v1/storage/files', ['bucketId' => 'default']) ->desc('Create File') ->groups(['api', 'storage']) ->label('scope', 'files.write') @@ -521,13 +543,14 @@ App::post('/v1/storage/buckets/:bucketId/files') $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } // Check bucket permissions when enforced - if ($bucket->getAttribute('permission') === 'bucket') { + $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; + if ($permissionBucket) { $validator = new Authorization('write'); if (!$validator->isValid($bucket->getWrite())) { throw new Exception('Unauthorized permissions', 401); @@ -555,14 +578,14 @@ App::post('/v1/storage/buckets/:bucketId/files') $file = $request->getFiles('file'); - /* + /** * Validators */ $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); $fileExt = new FileExt($allowedFileExtensions); $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); - if($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT',0)) { + if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) { throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500); } @@ -574,116 +597,234 @@ App::post('/v1/storage/buckets/:bucketId/files') } // Make sure we handle a single file and multiple files the same way - $file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + + $contentRange = $request->getHeader('content-range'); + $fileId = $fileId === 'unique()' ? $dbForProject->getId() : $fileId; + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $size = $request->getContentRangeSize(); + $fileId = $request->getHeader('x-appwrite-id', $fileId); + if (is_null($start) || is_null($end) || is_null($size)) { + throw new Exception('Invalid content-range header', 400); + } + + if ($end === $size) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($size / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)) + 1; + } + } // Check if file type is allowed (feature for project settings?) - if (!empty($allowedFileExtensions) && !$fileExt->isValid($file['name'])) { + if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { throw new Exception('File extension not allowed', 400); } - if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit + if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit throw new Exception('File size not allowed', 400); } $device = Storage::getDevice('files'); - if (!$upload->isValid($file['tmp_name'])) { + if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $size = $device->getFileSize($file['tmp_name']); - $path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION)); - $path = $bucket->getId() . DIRECTORY_SEPARATOR . $path; - - if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move' - throw new Exception('Failed moving file', 500); + $size ??= $device->getFileSize($fileTmpName); + $path = $device->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($device->getRoot(), $device->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + + $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + if ($chunk === -1) { + $chunk = $chunks; + } } - $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks); + if (empty($chunksUploaded)) { + throw new Exception('Failed uploading file', 500); + } - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) { - $antivirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; + $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; + if ($chunksUploaded === $chunks) { + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) { + $antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)); - - if (!$antivirus->fileScan($path)) { - $device->delete($path); - throw new Exception('Invalid file', 403); - } - } - - // Compression - $data = $device->read($path); - if($size <= APP_LIMIT_COMPRESSION) { - $compressor = new GZIP(); - $data = $compressor->compress($data); - } - - if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { - $key = App::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $tag = null; - $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); - } - - if (!$device->write($path, $data, $mimeType)) { - throw new Exception('Failed to save file', 500); - } - - $sizeActual = $device->getFileSize($path); - - $fileId = ($fileId == 'unique()') ? $dbForProject->getId() : $fileId; - $data = [ - '$id' => $fileId, - '$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user - '$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user - 'dateCreated' => \time(), - 'bucketId' => $bucket->getId(), - 'name' => $file['name'], - 'path' => $path, - 'signature' => $device->getFileHash($path), - 'mimeType' => $mimeType, - 'sizeOriginal' => $size, - 'sizeActual' => $sizeActual, - 'algorithm' => empty($compressor) ? '' : $compressor->getName(), - 'comment' => '', - 'search' => implode(' ', [$fileId, $file['name'] ?? '',]), - ]; - - if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { - $data['openSSLVersion'] = '1'; - $data['openSSLCipher'] = OpenSSL::CIPHER_AES_128_GCM; - $data['openSSLTag'] = \bin2hex($tag ?? ''); - $data['openSSLIV'] = \bin2hex($iv); - } - - try { - if($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function() use ($dbForProject, $bucket, $data) { - return $dbForProject->createDocument('bucket_' . $bucket->getId(), new Document($data)); - }); - } else { - $file = $dbForProject->createDocument('bucket_' . $bucket->getId(), new Document($data)); + + if (!$antiVirus->fileScan($path)) { + $device->delete($path); + throw new Exception('Invalid file', 403); + } } - } - catch (StructureException $exception) { - throw new Exception($exception->getMessage(), 400); - } - catch (DuplicateException $exception) { - throw new Exception('Document already exists', 409); - } + $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + $data = ''; + // Compression + if ($size <= APP_STORAGE_READ_BUFFER) { + $data = $device->read($path); + $compressor = new GZIP(); + $data = $compressor->compress($data); + } + if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) { + if (empty($data)) { + $data = $device->read($path); + } + $key = App::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if (!empty($data)) { + if (!$device->write($path, $data, $mimeType)) { + throw new Exception('Failed to save file', 500); + } + } + + $sizeActual = $device->getFileSize($path); + + $algorithm = empty($compressor) ? '' : $compressor->getName(); + $fileHash = $device->getFileHash($path); + + if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + try { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$read' => $read, + '$write' => $write, + 'dateCreated' => \time(), + 'bucketId' => $bucket->getId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $size, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName,]), + ]); + if ($permissionBucket) { + $file = Authorization::skip(function() use ($dbForProject, $bucketId, $doc) { + return $dbForProject->createDocument('bucket_' . $bucketId, $doc); + }); + } else { + $file = $dbForProject->createDocument('bucket_' . $bucketId, $doc); + } + } else { + $file = $file + ->setAttribute('$read', $read) + ->setAttribute('$write', $write) + ->setAttribute('signature', $fileHash) + ->setAttribute('mimeType', $mimeType) + ->setAttribute('sizeActual', $sizeActual) + ->setAttribute('algorithm', $algorithm) + ->setAttribute('openSSLVersion', $openSSLVersion) + ->setAttribute('openSSLCipher', $openSSLCipher) + ->setAttribute('openSSLTag', $openSSLTag) + ->setAttribute('openSSLIV', $openSSLIV) + ->setAttribute('chunksUploaded', $chunksUploaded); + + if ($permissionBucket) { + $file = Authorization::skip(function() use ($dbForProject, $bucketId, $fileId, $file) { + return $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + }); + } else { + $file = $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + } + + } + } + catch (StructureException $exception) { + throw new Exception($exception->getMessage(), 400); + } + catch (DuplicateException $exception) { + throw new Exception('Document already exists', 409); + } + } else { + try { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$read' => $read, + '$write' => $write, + 'dateCreated' => \time(), + 'bucketId' => $bucket->getId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $size, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName,]), + ]); + if ($permissionBucket) { + $file = Authorization::skip(function() use ($dbForProject, $bucketId, $doc) { + return $dbForProject->createDocument('bucket_' . $bucketId, $doc); + }); + } else { + $file = $dbForProject->createDocument('bucket_' . $bucketId, $doc); + } + } else { + $file = $file + ->setAttribute('chunksUploaded', $chunksUploaded); + + if ($permissionBucket) { + $file = Authorization::skip(function() use ($dbForProject, $bucketId, $fileId, $file) { + return $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + }); + } else { + $file = $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + } + } + } + catch (StructureException $exception) { + throw new Exception($exception->getMessage(), 400); + } + catch (DuplicateException $exception) { + throw new Exception('Document already exists', 409); + } + } $audits ->setParam('event', 'storage.files.create') - ->setParam('resource', 'file/'.$file->getId()) + ->setParam('resource', 'storage/files/' . $file->getId()) ; $usage - ->setParam('storage', $sizeActual) + ->setParam('storage', $sizeActual ?? 0) ->setParam('storage.files.create', 1) ->setParam('bucketId', $bucketId) ; @@ -724,7 +865,7 @@ App::get('/v1/storage/buckets/:bucketId/files') $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } @@ -739,12 +880,12 @@ App::get('/v1/storage/buckets/:bucketId/files') $queries = [new Query('bucketId', Query::TYPE_EQUAL, [$bucketId])]; - if($search) { + if ($search) { $queries[] = [new Query('name', Query::TYPE_SEARCH, [$search])]; } if (!empty($cursor)) { - if($bucket->getAttribute('permission') ==='bucket') { + if ($bucket->getAttribute('permission') ==='bucket') { $cursorFile = Authorization::skip(function() use ($dbForProject, $bucket, $cursor) { return $dbForProject->getDocument('bucket_' . $bucket->getId(), $cursor); }); @@ -808,7 +949,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId') $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } @@ -889,7 +1030,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') } $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } @@ -913,7 +1054,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT'; // 45 days cache $key = \md5($fileId.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output); - if($bucket->getAttribute('permission')==='bucket') { + if ($bucket->getAttribute('permission')==='bucket') { // skip authorization $file = Authorization::skip(function () use ($dbForProject, $bucketId, $fileId) { return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); @@ -948,8 +1089,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception('File not found', 404); } - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE.'/app-'.$project->getId())); // Limit file number or size - $data = $cache->load($key, 60 * 60 * 24 * 30 * 3 /* 3 months */); + $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-' . $project->getId())); // Limit file number or size + $data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */); if ($data) { $output = (empty($output)) ? $type : $output; @@ -968,7 +1109,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), 0, \hex2bin($file->getAttribute('openSSLIV')), \hex2bin($file->getAttribute('openSSLTag')) @@ -988,11 +1129,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') } if (!empty($background)) { - $image->setBackground('#'.$background); + $image->setBackground('#' . $background); } - - if (!empty($borderWidth) ) { - $image->setBorder($borderWidth, '#'.$borderColor); + + if (!empty($borderWidth)) { + $image->setBorder($borderWidth, '#' . $borderColor); } if (!empty($borderRadius)) { @@ -1038,20 +1179,21 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->label('sdk.methodType', 'location') ->param('bucketId', null, new UID(), 'Storage bucket ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') + ->inject('request') ->inject('response') ->inject('dbForProject') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $response, $dbForProject, $usage, $mode) { + ->action(function ($bucketId, $fileId, $request, $response, $dbForProject, $usage, $mode) { + /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ - /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Stats\Stats $usage */ /** @var string $mode */ $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } @@ -1078,26 +1220,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $path = $file->getAttribute('path', ''); - if (!\file_exists($path)) { - throw new Exception('File not found in '.$path, 404); - } - $device = Storage::getDevice('files'); - - $source = $device->read($path); - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - if(!empty($file->getAttribute('algorithm', ''))) { - $compressor = new GZIP(); - $source = $compressor->decompress($source); + + if (!$device->exists($path)) { + throw new Exception('File not found in ' . $path, 404); } $usage @@ -1105,14 +1231,83 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->setParam('bucketId', $bucketId) ; - // Response $response ->setContentType($file->getAttribute('mimeType')) - ->addHeader('Content-Disposition', 'attachment; filename="'.$file->getAttribute('name', '').'"') - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->addHeader('X-Peak', \memory_get_peak_usage()) - ->send($source) + ->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"') ; + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null) { + $end = min(($start + MAX_OUTPUT_CHUNK_SIZE-1), ($size - 1)); + } + + if ($unit !== 'bytes' || $start >= $end || $end >= $size) { + throw new Exception('Invalid range', 416); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $device->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + if (!empty($file->getAttribute('algorithm', ''))) { + if (empty($source)) { + $source = $device->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + } + + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + } + $response->send($source); + } + + if (!empty($rangeHeader)) { + $response->send($device->read($path, $start, ($end - $start + 1))); + } + + if ($size > APP_STORAGE_READ_BUFFER) { + $response->addHeader('Content-Length', $device->getFileSize($path)); + for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $device->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($device->read($path)); + } }); App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') @@ -1130,18 +1325,20 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') ->inject('response') + ->inject('request') ->inject('dbForProject') ->inject('usage') ->inject('mode') - ->action(function ($bucketId, $fileId, $response, $dbForProject, $usage, $mode) { + ->action(function ($bucketId, $fileId, $response, $request, $dbForProject, $usage, $mode) { /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Database $dbForProject */ + /** @var Utopia\Swoole\Request $request */ + /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Stats\Stats $usage */ /** @var string $mode */ $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } @@ -1171,7 +1368,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $path = $file->getAttribute('path', ''); if (!\file_exists($path)) { - throw new Exception('File not found in '.$path, 404); + throw new Exception('File not found in ' . $path, 404); } $compressor = new GZIP(); @@ -1183,36 +1380,91 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $contentType = $file->getAttribute('mimeType'); } - $source = $device->read($path); + $response + ->setContentType($contentType) + ->addHeader('Content-Security-Policy', 'script-src none;') + ->addHeader('X-Content-Type-Options', 'nosniff') + ->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"') + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache + ->addHeader('X-Peak', \memory_get_peak_usage()) + ; + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null) { + $end = min(($start + 2000000-1), ($size - 1)); + } + + if ($unit != 'bytes' || $start >= $end || $end >= $size) { + throw new Exception('Invalid range', 416); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', "bytes $start-$end/$size") + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $device->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), 0, \hex2bin($file->getAttribute('openSSLIV')), \hex2bin($file->getAttribute('openSSLTag')) ); } - $output = $compressor->decompress($source); - $fileName = $file->getAttribute('name', ''); + if (!empty($file->getAttribute('algorithm', ''))) { + if (empty($source)) { + $source = $device->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + } $usage ->setParam('storage.files.read', 1) ->setParam('bucketId', $bucketId) ; - $response - ->setContentType($contentType) - ->addHeader('Content-Security-Policy', 'script-src none;') - ->addHeader('X-Content-Type-Options', 'nosniff') - ->addHeader('Content-Disposition', 'inline; filename="'.$fileName.'"') - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache - ->addHeader('X-Peak', \memory_get_peak_usage()) - ->send($output) - ; + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + } + $response->send($source); + } + + if (!empty($rangeHeader)) { + $response->send($device->read($path, $start, ($end - $start + 1))); + } + + $size = $device->getFileSize($path); + if ($size > APP_STORAGE_READ_BUFFER) { + $response->addHeader('Content-Length', $device->getFileSize($path)); + for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $device->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($device->read($path)); + } }); App::put('/v1/storage/buckets/:bucketId/files/:fileId') @@ -1266,7 +1518,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') } } - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } @@ -1349,7 +1601,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty() + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) { throw new Exception('Bucket not found', 404); } @@ -1376,8 +1628,15 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') $device = Storage::getDevice('files'); - if ($device->delete($file->getAttribute('path', ''))) { - if($bucket->getAttribute('permission') === 'bucket') { + $deviceDeleted = false; + if($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) { + $deviceDeleted = $device->abort($file->getAttribute('path')); + } else { + $deviceDeleted = $device->delete($file->getAttribute('path')); + } + + if ($deviceDeleted) { + if ($bucket->getAttribute('permission') === 'bucket') { $deleted = Authorization::skip(function() use ($dbForProject, $fileId, $bucketId) { return $dbForProject->deleteDocument('bucket_' . $bucketId, $fileId); }); @@ -1387,8 +1646,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') if (!$deleted) { throw new Exception('Failed to remove file from DB', 500); } + } else { + throw new Exception('Failed to delete file from device', 500); } - + $audits ->setParam('event', 'storage.files.delete') ->setParam('resource', 'file/'.$file->getId()) @@ -1538,7 +1799,7 @@ App::get('/v1/storage/:bucketId/usage') $bucket = $dbForProject->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } diff --git a/app/controllers/general.php b/app/controllers/general.php index 7f82c36a30..2664f544ee 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -411,6 +411,7 @@ App::error(function ($error, $utopia, $request, $response, $layout, $project, $l case 404: // Error allowed publicly case 409: // Error allowed publicly case 412: // Error allowed publicly + case 416: // Error allowed publicly case 429: // Error allowed publicly case 501: // Error allowed publicly case 503: // Error allowed publicly diff --git a/app/http.php b/app/http.php index a73daa2562..4414ddf728 100644 --- a/app/http.php +++ b/app/http.php @@ -21,7 +21,7 @@ use Utopia\Logger\Log\User; $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); -$payloadSize = max(4000000 /* 4mb */, App::getEnv('_APP_STORAGE_LIMIT', 10000000 /* 10mb */)); +$payloadSize = 6 * (1024 * 1024); // 6MB $http ->set([ diff --git a/app/init.php b/app/init.php index ddcbc3f97c..9cd7eeae77 100644 --- a/app/init.php +++ b/app/init.php @@ -78,6 +78,7 @@ const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_CACHE = '/storage/cache'; const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; +const APP_STORAGE_READ_BUFFER = 20 * (1024 * 1024); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` const APP_SOCIAL_TWITTER = 'https://twitter.com/appwrite'; const APP_SOCIAL_TWITTER_HANDLE = 'appwrite'; const APP_SOCIAL_FACEBOOK = 'https://www.facebook.com/appwrite.io'; @@ -121,6 +122,8 @@ const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; const APP_AUTH_TYPE_KEY = 'Key'; const APP_AUTH_TYPE_ADMIN = 'Admin'; +// Response related +const MAX_OUTPUT_CHUNK_SIZE = 2*1024*1024; // 2MB $register = new Registry(); diff --git a/app/views/console/storage/bucket.phtml b/app/views/console/storage/bucket.phtml index 8f31b24d9d..ce87a5de96 100644 --- a/app/views/console/storage/bucket.phtml +++ b/app/views/console/storage/bucket.phtml @@ -158,7 +158,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0); -
+
File Preview
@@ -188,7 +188,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
-   +  
@@ -196,8 +196,11 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0); - - +
+ + +
+ incomplete diff --git a/composer.lock b/composer.lock index 7dd9f94770..2ae45c3928 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ab493f0a7f01a1105f8bc5caaf9b928b", + "content-hash": "d9d66a3e700bc3936fe9a09f3c1b38f9", "packages": [ { "name": "adhocore/jwt", @@ -2141,16 +2141,16 @@ }, { "name": "utopia-php/database", - "version": "0.14.0", + "version": "0.14.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "2f2527bb080cf578fba327ea2ec637064561d403" + "reference": "ecc143f2cfe16b23675407035c6b5375ba263285" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/2f2527bb080cf578fba327ea2ec637064561d403", - "reference": "2f2527bb080cf578fba327ea2ec637064561d403", + "url": "https://api.github.com/repos/utopia-php/database/zipball/ecc143f2cfe16b23675407035c6b5375ba263285", + "reference": "ecc143f2cfe16b23675407035c6b5375ba263285", "shasum": "" }, "require": { @@ -2198,9 +2198,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.14.0" + "source": "https://github.com/utopia-php/database/tree/0.14.1" }, - "time": "2022-01-21T16:34:34+00:00" + "time": "2022-01-25T13:01:20+00:00" }, { "name": "utopia-php/domains", @@ -3126,23 +3126,23 @@ }, { "name": "composer/pcre", - "version": "1.0.0", + "version": "1.0.1", "source": { "type": "git", "url": "https://github.com/composer/pcre.git", - "reference": "3d322d715c43a1ac36c7fe215fa59336265500f2" + "reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/pcre/zipball/3d322d715c43a1ac36c7fe215fa59336265500f2", - "reference": "3d322d715c43a1ac36c7fe215fa59336265500f2", + "url": "https://api.github.com/repos/composer/pcre/zipball/67a32d7d6f9f560b726ab25a061b38ff3a80c560", + "reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560", "shasum": "" }, "require": { "php": "^5.3.2 || ^7.0 || ^8.0" }, "require-dev": { - "phpstan/phpstan": "^1", + "phpstan/phpstan": "^1.3", "phpstan/phpstan-strict-rules": "^1.1", "symfony/phpunit-bridge": "^4.2 || ^5" }, @@ -3177,7 +3177,7 @@ ], "support": { "issues": "https://github.com/composer/pcre/issues", - "source": "https://github.com/composer/pcre/tree/1.0.0" + "source": "https://github.com/composer/pcre/tree/1.0.1" }, "funding": [ { @@ -3193,7 +3193,7 @@ "type": "tidelift" } ], - "time": "2021-12-06T15:17:27+00:00" + "time": "2022-01-21T20:24:37+00:00" }, { "name": "composer/semver", @@ -3697,6 +3697,9 @@ "require": { "php": "^7.1 || ^8.0" }, + "replace": { + "myclabs/deep-copy": "self.version" + }, "require-dev": { "doctrine/collections": "^1.0", "doctrine/common": "^2.6", @@ -6662,5 +6665,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.1.0" } diff --git a/docs/references/storage/create-file.md b/docs/references/storage/create-file.md index 7fd6c5a492..c52a7c4e04 100644 --- a/docs/references/storage/create-file.md +++ b/docs/references/storage/create-file.md @@ -1 +1,7 @@ -Create a new file. The user who creates the file will automatically be assigned to read and write access unless he has passed custom values for read and write arguments. \ No newline at end of file +Create a new file. The user that creates the file will automatically be granted read and write permissions unless they have passed custom values for read and write permissions. + +Larger files should be uploaded using multiple requests with the [content-range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range) header to send a partial request with a maximum supported chunk of `5MB`. The `content-range` header values should always be in bytes. + +When the first request is sent, the server will return the **File** object, and the subsequent part request must include the file's **id** in `x-appwrite-upload-id` header to allow the server to know that the partial upload is for the existing file and not for a new one. + +If you're creating a new file using one the Appwrite SDKs, all the chunking logic will be managed by the SDK internally. diff --git a/src/Appwrite/Utopia/Response/Model/File.php b/src/Appwrite/Utopia/Response/Model/File.php index a263f1ed9b..f41557b23b 100644 --- a/src/Appwrite/Utopia/Response/Model/File.php +++ b/src/Appwrite/Utopia/Response/Model/File.php @@ -66,6 +66,18 @@ class File extends Model 'default' => 0, 'example' => 17890, ]) + ->addRule('chunksTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of chunks available', + 'default' => 0, + 'example' => 17890, + ]) + ->addRule('chunksUploaded', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of chunks uploaded', + 'default' => 0, + 'example' => 17890, + ]) ; } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index c2452c08bd..b28e0ff223 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -292,6 +292,41 @@ class FunctionsCustomServerTest extends Scope $this->assertIsInt($tag['body']['dateCreated']); $this->assertEquals('php index.php', $tag['body']['command']); $this->assertGreaterThan(10000, $tag['body']['size']); + + /** + * Test for Large Code File SUCCESS + */ + $source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz'); + $chunkSize = 5*1024*1024; + $handle = @fopen($source, "rb"); + $mimeType = 'application/x-gzip'; + $counter = 0; + $size = filesize($source); + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'] + ]; + $id = ''; + while (!feof($handle)) { + $curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz'); + $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; + if(!empty($id)) { + $headers['x-appwrite-id'] = $id; + } + $largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [ + 'command' => 'php index.php', + 'code' => $curlFile, + ]); + $counter++; + $id = $largeTag['body']['$id']; + } + @fclose($handle); + + $this->assertEquals(201, $largeTag['headers']['status-code']); + $this->assertNotEmpty($largeTag['body']['$id']); + $this->assertIsInt($largeTag['body']['dateCreated']); + $this->assertEquals('php index.php', $largeTag['body']['command']); + $this->assertGreaterThan(10000, $largeTag['body']['size']); /** * Test for FAILURE @@ -342,9 +377,9 @@ class FunctionsCustomServerTest extends Scope ], $this->getHeaders())); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); /** * Test search queries @@ -357,9 +392,9 @@ class FunctionsCustomServerTest extends Scope ])); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ @@ -370,9 +405,9 @@ class FunctionsCustomServerTest extends Scope ])); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ @@ -383,9 +418,9 @@ class FunctionsCustomServerTest extends Scope ])); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); return $data; diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index e6eabe9384..87d45e579f 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -65,24 +65,48 @@ trait StorageBase ]); $this->assertEquals(201, $bucket2['headers']['status-code']); $this->assertNotEmpty($bucket2['body']['$id']); + + /** + * Chunked Upload + */ - $file2 = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge([ + $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $totalSize = \filesize($source); + $chunkSize = 5*1024*1024; + $handle = @fopen($source, "rb"); + $fileId = 'unique()'; + $mimeType = mime_content_type($source); + $counter = 0; + $size = filesize($source); + $headers = [ 'content-type' => 'multipart/form-data', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'fileId' => 'unique()', - 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4'), 'video/mp4', 'large-file.mp4'), - 'read' => ['role:all'], - 'write' => ['role:all'], - ]); - - $this->assertEquals(201, $file2['headers']['status-code']); - $this->assertNotEmpty($file2['body']['$id']); - $this->assertIsInt($file2['body']['dateCreated']); - $this->assertEquals('large-file.mp4', $file2['body']['name']); - $this->assertEquals('video/mp4', $file2['body']['mimeType']); - $this->assertEquals(23660615, $file2['body']['sizeOriginal']); - $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $file2['body']['signature']); // should validate that the file is not encrypted + 'x-appwrite-project' => $this->getProject()['$id'] + ]; + $id = ''; + while (!feof($handle)) { + $curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'large-file.mp4'); + $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; + if(!empty($id)) { + $headers['x-appwrite-id'] = $id; + } + $largeFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => $curlFile, + 'read' => ['role:all'], + 'write' => ['role:all'], + ]); + $counter++; + $id = $largeFile['body']['$id']; + } + @fclose($handle); + + $this->assertEquals(201, $largeFile['headers']['status-code']); + $this->assertNotEmpty($largeFile['body']['$id']); + $this->assertIsInt($largeFile['body']['dateCreated']); + $this->assertEquals('large-file.mp4', $largeFile['body']['name']); + $this->assertEquals('video/mp4', $largeFile['body']['mimeType']); + $this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']); + $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted /** * Test for FAILURE unknown Bucket @@ -133,7 +157,7 @@ trait StorageBase $this->assertEquals(400, $res['headers']['status-code']); $this->assertEquals('File extension not allowed', $res['body']['message']); - return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $file2['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']]; + return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $largeFile['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']]; } /** @@ -263,6 +287,49 @@ trait StorageBase $this->assertEquals('image/png', $file5['headers']['content-type']); $this->assertNotEmpty($file5['body']); + // Test ranged download + $file51 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=0-99', + ], $this->getHeaders())); + + $path = __DIR__ . '/../../../resources/logo.png'; + $originalChunk = \file_get_contents($path, false, null, 0, 100); + + $this->assertEquals(206, $file51['headers']['status-code']); + $this->assertEquals('attachment; filename="logo.png"', $file51['headers']['content-disposition']); + $this->assertEquals('image/png', $file51['headers']['content-type']); + $this->assertNotEmpty($file51['body']); + $this->assertEquals($originalChunk, $file51['body']); + + // Test ranged download - with invalid range + $file52 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=0-', + ], $this->getHeaders())); + + $this->assertEquals(206, $file52['headers']['status-code']); + + // Test ranged download - with invalid range + $file53 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=988', + ], $this->getHeaders())); + + $this->assertEquals(416, $file53['headers']['status-code']); + + // Test ranged download - with invalid range + $file54 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=-988', + ], $this->getHeaders())); + + $this->assertEquals(416, $file54['headers']['status-code']); + $file6 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/view', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/resources/functions/php-large.tar.gz b/tests/resources/functions/php-large.tar.gz new file mode 100644 index 0000000000..7ca5857029 Binary files /dev/null and b/tests/resources/functions/php-large.tar.gz differ