Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-22 08:28:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1383 from lohanidamodar/feat-large-file

Browse source
This commit is contained in:
Damodar Lohani 2022-01-30 11:56:43 +05:45 committed by GitHub
commit ccf1fd5501
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 708 additions and 235 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
app/config/collections.php
View file

@ -1769,6 +1769,28 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => 'chunksTotal',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'chunksUploaded',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'search',
'type' => Database::VAR_STRING,

100
app/controllers/api/functions.php
View file

@ -475,42 +475,102 @@ App::post('/v1/functions/:functionId/tags')
}
// Make sure we handle a single file and multiple files the same way
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed
throw new Exception('File type not allowed', 400);
}
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
$contentRange = $request->getHeader('content-range');
$tagId = $dbForProject->getId();
$chunk = 1;
$chunks = 1;
if (!empty($contentRange)) {
$start = $request->getContentRangeStart();
$end = $request->getContentRangeEnd();
$size = $request->getContentRangeSize();
$tagId = $request->getHeader('x-appwrite-id', $tagId);
if(is_null($start) || is_null($end) || is_null($size)) {
throw new Exception('Invalid content-range header', 400);
}
if ($end == $size) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($size / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start)) + 1;
}
}
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
throw new Exception('File size not allowed', 400);
}
if (!$upload->isValid($file['tmp_name'])) {
if (!$upload->isValid($fileTmpName)) {
throw new Exception('Invalid file', 403);
}
// Save to storage
$size = $device->getFileSize($file['tmp_name']);
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
$size ??= $device->getFileSize($fileTmpName);
$path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION));
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
$tag = $dbForProject->getDocument('tags', $tagId);
if(!$tag->isEmpty()) {
$chunks = $tag->getAttribute('chunksTotal', 1);
if($chunk == -1) {
$chunk = $chunks;
}
}
$chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
if (empty($chunksUploaded)) {
throw new Exception('Failed moving file', 500);
}
$tagId = $dbForProject->getId();
$tag = $dbForProject->createDocument('tags', new Document([
'$id' => $tagId,
'$read' => [],
'$write' => [],
'functionId' => $function->getId(),
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => $size,
'search' => implode(' ', [$tagId, $command]),
]));
if($chunksUploaded == $chunks) {
$size = $device->getFileSize($path);
if ($tag->isEmpty()) {
$tag = $dbForProject->createDocument('tags', new Document([
'$id' => $tagId,
'$read' => [],
'$write' => [],
'functionId' => $function->getId(),
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => $size,
'search' => implode(' ', [$tagId, $command]),
]));
} else {
$tag = $dbForProject->updateDocument('tags', $tagId, $tag->setAttribute('size', $size));
}
} else {
if($tag->isEmpty()) {
$tag = $dbForProject->createDocument('tags', new Document([
'$id' => $tagId,
'$read' => [],
'$write' => [],
'functionId' => $function->getId(),
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => 0,
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$tagId, $command]),
]));
} else {
$tag = $dbForProject->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded));
}
}
$usage
->setParam('storage', $tag->getAttribute('size', 0))

599
app/controllers/api/storage.php
View file

@ -2,19 +2,13 @@
use Appwrite\Auth\Auth;
use Utopia\App;
use Utopia\Exception;
use Utopia\Validator\ArrayList;
use Utopia\Validator\WhiteList;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Filesystem;
use Appwrite\ClamAV\Network;
use Utopia\Database\Validator\Authorization;
use Appwrite\Database\Validator\CustomId;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Exception;
use Utopia\Database\Validator\UID;
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\File;
@ -24,11 +18,17 @@ use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Image\Image;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\Utopia\Response;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Exception\Duplicate;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Validator\Integer;
use Utopia\Database\Query;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\Validator\Range;
use Utopia\Database\Validator\Permissions;
use Utopia\Storage\Validator\FileExt;
use Utopia\Database\Exception\Duplicate as DuplicateException;
@ -55,7 +55,7 @@ App::post('/v1/storage/buckets')
->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true)
->param('enabled', true, new Boolean(), 'Is bucket enabled?', true)
->param('adapter', 'local', new WhiteList(['local']), 'Storage adapter.', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true)
->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
->inject('response')
->inject('dbForProject')
@ -225,6 +225,28 @@ App::post('/v1/storage/buckets')
'array' => false,
'filters' => [],
]),
new Document([
'$id' => 'chunksTotal',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
]),
new Document([
'$id' => 'chunksUploaded',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
]),
new Document([
'$id' => 'search',
'type' => Database::VAR_STRING,
@ -237,13 +259,6 @@ App::post('/v1/storage/buckets')
'filters' => [],
]),
], [
new Document([
'$id' => '_key_bucket',
'type' => Database::INDEX_KEY,
'attributes' => ['bucketId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
]),
new Document([
'$id' => '_key_search',
'type' => Database::INDEX_FULLTEXT,
@ -251,6 +266,13 @@ App::post('/v1/storage/buckets')
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
]),
new Document([
'$id' => '_key_bucket',
'type' => Database::INDEX_KEY,
'attributes' => ['bucketId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
]),
]);
$bucket = $dbForProject->createDocument('buckets', new Document([
@ -379,7 +401,7 @@ App::put('/v1/storage/buckets/:bucketId')
->param('maximumFileSize', null, new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self hosted version you can change the limit by changing _APP_STORAGE_LIMIT environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true)
->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true)
->param('enabled', true, new Boolean(), 'Is bucket enabled?', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true)
->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
->inject('response')
->inject('dbForProject')
@ -398,7 +420,7 @@ App::put('/v1/storage/buckets/:bucketId')
}
$read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions
$write ??= $bucket->getAttribute('$write',[]); // By default inherit write permissions
$write ??= $bucket->getAttribute('$write', []); // By default inherit write permissions
$maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int)App::getEnv('_APP_STORAGE_LIMIT', 0));
$allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []);
$enabled ??= $bucket->getAttribute('enabled', true);
@ -484,7 +506,7 @@ App::delete('/v1/storage/buckets/:bucketId')
});
App::post('/v1/storage/buckets/:bucketId/files')
->alias('/v1/storage/files',['bucketId' => 'default'])
->alias('/v1/storage/files', ['bucketId' => 'default'])
->desc('Create File')
->groups(['api', 'storage'])
->label('scope', 'files.write')
@ -521,13 +543,14 @@ App::post('/v1/storage/buckets/:bucketId/files')
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
// Check bucket permissions when enforced
if ($bucket->getAttribute('permission') === 'bucket') {
$permissionBucket = $bucket->getAttribute('permission') === 'bucket';
if ($permissionBucket) {
$validator = new Authorization('write');
if (!$validator->isValid($bucket->getWrite())) {
throw new Exception('Unauthorized permissions', 401);
@ -555,14 +578,14 @@ App::post('/v1/storage/buckets/:bucketId/files')
$file = $request->getFiles('file');
/*
/**
* Validators
*/
$allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []);
$fileExt = new FileExt($allowedFileExtensions);
$maximumFileSize = $bucket->getAttribute('maximumFileSize', 0);
if($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT',0)) {
if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) {
throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500);
}
@ -574,116 +597,234 @@ App::post('/v1/storage/buckets/:bucketId/files')
}
// Make sure we handle a single file and multiple files the same way
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$contentRange = $request->getHeader('content-range');
$fileId = $fileId === 'unique()' ? $dbForProject->getId() : $fileId;
$chunk = 1;
$chunks = 1;
if (!empty($contentRange)) {
$start = $request->getContentRangeStart();
$end = $request->getContentRangeEnd();
$size = $request->getContentRangeSize();
$fileId = $request->getHeader('x-appwrite-id', $fileId);
if (is_null($start) || is_null($end) || is_null($size)) {
throw new Exception('Invalid content-range header', 400);
}
if ($end === $size) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($size / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start)) + 1;
}
}
// Check if file type is allowed (feature for project settings?)
if (!empty($allowedFileExtensions) && !$fileExt->isValid($file['name'])) {
if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) {
throw new Exception('File extension not allowed', 400);
}
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
throw new Exception('File size not allowed', 400);
}
$device = Storage::getDevice('files');
if (!$upload->isValid($file['tmp_name'])) {
if (!$upload->isValid($fileTmpName)) {
throw new Exception('Invalid file', 403);
}
// Save to storage
$size = $device->getFileSize($file['tmp_name']);
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
$path = $bucket->getId() . DIRECTORY_SEPARATOR . $path;
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
throw new Exception('Failed moving file', 500);
$size ??= $device->getFileSize($fileTmpName);
$path = $device->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$path = str_ireplace($device->getRoot(), $device->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root
$file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId);
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('chunksTotal', 1);
if ($chunk === -1) {
$chunk = $chunks;
}
}
$mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption
$chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
if (empty($chunksUploaded)) {
throw new Exception('Failed uploading file', 500);
}
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) {
$antivirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? [];
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
if ($chunksUploaded === $chunks) {
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) {
$antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310));
if (!$antivirus->fileScan($path)) {
$device->delete($path);
throw new Exception('Invalid file', 403);
}
}
// Compression
$data = $device->read($path);
if($size <= APP_LIMIT_COMPRESSION) {
$compressor = new GZIP();
$data = $compressor->compress($data);
}
if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$tag = null;
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
if (!$device->write($path, $data, $mimeType)) {
throw new Exception('Failed to save file', 500);
}
$sizeActual = $device->getFileSize($path);
$fileId = ($fileId == 'unique()') ? $dbForProject->getId() : $fileId;
$data = [
'$id' => $fileId,
'$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user
'$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $file['name'],
'path' => $path,
'signature' => $device->getFileHash($path),
'mimeType' => $mimeType,
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => empty($compressor) ? '' : $compressor->getName(),
'comment' => '',
'search' => implode(' ', [$fileId, $file['name'] ?? '',]),
];
if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$data['openSSLVersion'] = '1';
$data['openSSLCipher'] = OpenSSL::CIPHER_AES_128_GCM;
$data['openSSLTag'] = \bin2hex($tag ?? '');
$data['openSSLIV'] = \bin2hex($iv);
}
try {
if($bucket->getAttribute('permission') === 'bucket') {
$file = Authorization::skip(function() use ($dbForProject, $bucket, $data) {
return $dbForProject->createDocument('bucket_' . $bucket->getId(), new Document($data));
});
} else {
$file = $dbForProject->createDocument('bucket_' . $bucket->getId(), new Document($data));
if (!$antiVirus->fileScan($path)) {
$device->delete($path);
throw new Exception('Invalid file', 403);
}
}
}
catch (StructureException $exception) {
throw new Exception($exception->getMessage(), 400);
}
catch (DuplicateException $exception) {
throw new Exception('Document already exists', 409);
}
$mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption
$data = '';
// Compression
if ($size <= APP_STORAGE_READ_BUFFER) {
$data = $device->read($path);
$compressor = new GZIP();
$data = $compressor->compress($data);
}
if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) {
if (empty($data)) {
$data = $device->read($path);
}
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
if (!empty($data)) {
if (!$device->write($path, $data, $mimeType)) {
throw new Exception('Failed to save file', 500);
}
}
$sizeActual = $device->getFileSize($path);
$algorithm = empty($compressor) ? '' : $compressor->getName();
$fileHash = $device->getFileHash($path);
if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) {
$openSSLVersion = '1';
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
$openSSLTag = \bin2hex($tag);
$openSSLIV = \bin2hex($iv);
}
try {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$read' => $read,
'$write' => $write,
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'search' => implode(' ', [$fileId, $fileName,]),
]);
if ($permissionBucket) {
$file = Authorization::skip(function() use ($dbForProject, $bucketId, $doc) {
return $dbForProject->createDocument('bucket_' . $bucketId, $doc);
});
} else {
$file = $dbForProject->createDocument('bucket_' . $bucketId, $doc);
}
} else {
$file = $file
->setAttribute('$read', $read)
->setAttribute('$write', $write)
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
->setAttribute('openSSLIV', $openSSLIV)
->setAttribute('chunksUploaded', $chunksUploaded);
if ($permissionBucket) {
$file = Authorization::skip(function() use ($dbForProject, $bucketId, $fileId, $file) {
return $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file);
});
} else {
$file = $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file);
}
}
}
catch (StructureException $exception) {
throw new Exception($exception->getMessage(), 400);
}
catch (DuplicateException $exception) {
throw new Exception('Document already exists', 409);
}
} else {
try {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$read' => $read,
'$write' => $write,
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $size,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$fileId, $fileName,]),
]);
if ($permissionBucket) {
$file = Authorization::skip(function() use ($dbForProject, $bucketId, $doc) {
return $dbForProject->createDocument('bucket_' . $bucketId, $doc);
});
} else {
$file = $dbForProject->createDocument('bucket_' . $bucketId, $doc);
}
} else {
$file = $file
->setAttribute('chunksUploaded', $chunksUploaded);
if ($permissionBucket) {
$file = Authorization::skip(function() use ($dbForProject, $bucketId, $fileId, $file) {
return $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file);
});
} else {
$file = $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file);
}
}
}
catch (StructureException $exception) {
throw new Exception($exception->getMessage(), 400);
}
catch (DuplicateException $exception) {
throw new Exception('Document already exists', 409);
}
}
$audits
->setParam('event', 'storage.files.create')
->setParam('resource', 'file/'.$file->getId())
->setParam('resource', 'storage/files/' . $file->getId())
;
$usage
->setParam('storage', $sizeActual)
->setParam('storage', $sizeActual ?? 0)
->setParam('storage.files.create', 1)
->setParam('bucketId', $bucketId)
;
@ -724,7 +865,7 @@ App::get('/v1/storage/buckets/:bucketId/files')
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
@ -739,12 +880,12 @@ App::get('/v1/storage/buckets/:bucketId/files')
$queries = [new Query('bucketId', Query::TYPE_EQUAL, [$bucketId])];
if($search) {
if ($search) {
$queries[] = [new Query('name', Query::TYPE_SEARCH, [$search])];
}
if (!empty($cursor)) {
if($bucket->getAttribute('permission') ==='bucket') {
if ($bucket->getAttribute('permission') ==='bucket') {
$cursorFile = Authorization::skip(function() use ($dbForProject, $bucket, $cursor) {
return $dbForProject->getDocument('bucket_' . $bucket->getId(), $cursor);
});
@ -808,7 +949,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
@ -889,7 +1030,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
}
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
@ -913,7 +1054,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT'; // 45 days cache
$key = \md5($fileId.$width.$height.$gravity.$quality.$borderWidth.$borderColor.$borderRadius.$opacity.$rotation.$background.$storage.$output);
if($bucket->getAttribute('permission')==='bucket') {
if ($bucket->getAttribute('permission')==='bucket') {
// skip authorization
$file = Authorization::skip(function () use ($dbForProject, $bucketId, $fileId) {
return $dbForProject->getDocument('bucket_' . $bucketId, $fileId);
@ -948,8 +1089,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
throw new Exception('File not found', 404);
}
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE.'/app-'.$project->getId())); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3 /* 3 months */);
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-' . $project->getId())); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */);
if ($data) {
$output = (empty($output)) ? $type : $output;
@ -968,7 +1109,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
@ -988,11 +1129,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
}
if (!empty($background)) {
$image->setBackground('#'.$background);
$image->setBackground('#' . $background);
}
if (!empty($borderWidth) ) {
$image->setBorder($borderWidth, '#'.$borderColor);
if (!empty($borderWidth)) {
$image->setBorder($borderWidth, '#' . $borderColor);
}
if (!empty($borderRadius)) {
@ -1038,20 +1179,21 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->label('sdk.methodType', 'location')
->param('bucketId', null, new UID(), 'Storage bucket ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
->inject('request')
->inject('response')
->inject('dbForProject')
->inject('usage')
->inject('mode')
->action(function ($bucketId, $fileId, $response, $dbForProject, $usage, $mode) {
->action(function ($bucketId, $fileId, $request, $response, $dbForProject, $usage, $mode) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForProject */
/** @var Utopia\Database\Database $dbForProject */
/** @var Appwrite\Stats\Stats $usage */
/** @var string $mode */
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
@ -1078,26 +1220,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
$path = $file->getAttribute('path', '');
if (!\file_exists($path)) {
throw new Exception('File not found in '.$path, 404);
}
$device = Storage::getDevice('files');
$source = $device->read($path);
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
);
}
if(!empty($file->getAttribute('algorithm', ''))) {
$compressor = new GZIP();
$source = $compressor->decompress($source);
if (!$device->exists($path)) {
throw new Exception('File not found in ' . $path, 404);
}
$usage
@ -1105,14 +1231,83 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->setParam('bucketId', $bucketId)
;
// Response
$response
->setContentType($file->getAttribute('mimeType'))
->addHeader('Content-Disposition', 'attachment; filename="'.$file->getAttribute('name', '').'"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
->send($source)
->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"')
;
$size = $file->getAttribute('sizeOriginal', 0);
$rangeHeader = $request->getHeader('range');
if (!empty($rangeHeader)) {
$start = $request->getRangeStart();
$end = $request->getRangeEnd();
$unit = $request->getRangeUnit();
if ($end === null) {
$end = min(($start + MAX_OUTPUT_CHUNK_SIZE-1), ($size - 1));
}
if ($unit !== 'bytes' || $start >= $end || $end >= $size) {
throw new Exception('Invalid range', 416);
}
$response
->addHeader('Accept-Ranges', 'bytes')
->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size)
->addHeader('Content-Length', $end - $start + 1)
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT);
}
$source = '';
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
$source = $device->read($path);
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
);
}
if (!empty($file->getAttribute('algorithm', ''))) {
if (empty($source)) {
$source = $device->read($path);
}
$compressor = new GZIP();
$source = $compressor->decompress($source);
}
if (!empty($source)) {
if (!empty($rangeHeader)) {
$response->send(substr($source, $start, ($end - $start + 1)));
}
$response->send($source);
}
if (!empty($rangeHeader)) {
$response->send($device->read($path, $start, ($end - $start + 1)));
}
if ($size > APP_STORAGE_READ_BUFFER) {
$response->addHeader('Content-Length', $device->getFileSize($path));
for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
$response->chunk(
$device->read(
$path,
($i * MAX_OUTPUT_CHUNK_SIZE),
min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE))
),
(($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size
);
}
} else {
$response->send($device->read($path));
}
});
App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
@ -1130,18 +1325,20 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
->inject('response')
->inject('request')
->inject('dbForProject')
->inject('usage')
->inject('mode')
->action(function ($bucketId, $fileId, $response, $dbForProject, $usage, $mode) {
->action(function ($bucketId, $fileId, $response, $request, $dbForProject, $usage, $mode) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForProject */
/** @var Utopia\Swoole\Request $request */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
/** @var string $mode */
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
@ -1171,7 +1368,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
$path = $file->getAttribute('path', '');
if (!\file_exists($path)) {
throw new Exception('File not found in '.$path, 404);
throw new Exception('File not found in ' . $path, 404);
}
$compressor = new GZIP();
@ -1183,36 +1380,91 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
$contentType = $file->getAttribute('mimeType');
}
$source = $device->read($path);
$response
->setContentType($contentType)
->addHeader('Content-Security-Policy', 'script-src none;')
->addHeader('X-Content-Type-Options', 'nosniff')
->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
;
$size = $file->getAttribute('sizeOriginal', 0);
$rangeHeader = $request->getHeader('range');
if (!empty($rangeHeader)) {
$start = $request->getRangeStart();
$end = $request->getRangeEnd();
$unit = $request->getRangeUnit();
if ($end === null) {
$end = min(($start + 2000000-1), ($size - 1));
}
if ($unit != 'bytes' || $start >= $end || $end >= $size) {
throw new Exception('Invalid range', 416);
}
$response
->addHeader('Accept-Ranges', 'bytes')
->addHeader('Content-Range', "bytes $start-$end/$size")
->addHeader('Content-Length', $end - $start + 1)
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT);
}
$source = '';
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
$source = $device->read($path);
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
);
}
$output = $compressor->decompress($source);
$fileName = $file->getAttribute('name', '');
if (!empty($file->getAttribute('algorithm', ''))) {
if (empty($source)) {
$source = $device->read($path);
}
$compressor = new GZIP();
$source = $compressor->decompress($source);
}
$usage
->setParam('storage.files.read', 1)
->setParam('bucketId', $bucketId)
;
$response
->setContentType($contentType)
->addHeader('Content-Security-Policy', 'script-src none;')
->addHeader('X-Content-Type-Options', 'nosniff')
->addHeader('Content-Disposition', 'inline; filename="'.$fileName.'"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
->send($output)
;
if (!empty($source)) {
if (!empty($rangeHeader)) {
$response->send(substr($source, $start, ($end - $start + 1)));
}
$response->send($source);
}
if (!empty($rangeHeader)) {
$response->send($device->read($path, $start, ($end - $start + 1)));
}
$size = $device->getFileSize($path);
if ($size > APP_STORAGE_READ_BUFFER) {
$response->addHeader('Content-Length', $device->getFileSize($path));
for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
$response->chunk(
$device->read(
$path,
($i * MAX_OUTPUT_CHUNK_SIZE),
min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE))
),
(($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size
);
}
} else {
$response->send($device->read($path));
}
});
App::put('/v1/storage/buckets/:bucketId/files/:fileId')
@ -1266,7 +1518,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
}
}
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
@ -1349,7 +1601,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()
if ($bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN )) {
throw new Exception('Bucket not found', 404);
}
@ -1376,8 +1628,15 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
$device = Storage::getDevice('files');
if ($device->delete($file->getAttribute('path', ''))) {
if($bucket->getAttribute('permission') === 'bucket') {
$deviceDeleted = false;
if($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) {
$deviceDeleted = $device->abort($file->getAttribute('path'));
} else {
$deviceDeleted = $device->delete($file->getAttribute('path'));
}
if ($deviceDeleted) {
if ($bucket->getAttribute('permission') === 'bucket') {
$deleted = Authorization::skip(function() use ($dbForProject, $fileId, $bucketId) {
return $dbForProject->deleteDocument('bucket_' . $bucketId, $fileId);
});
@ -1387,8 +1646,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
if (!$deleted) {
throw new Exception('Failed to remove file from DB', 500);
}
} else {
throw new Exception('Failed to delete file from device', 500);
}
$audits
->setParam('event', 'storage.files.delete')
->setParam('resource', 'file/'.$file->getId())
@ -1538,7 +1799,7 @@ App::get('/v1/storage/:bucketId/usage')
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}

1
app/controllers/general.php
View file

@ -411,6 +411,7 @@ App::error(function ($error, $utopia, $request, $response, $layout, $project, $l
case 404: // Error allowed publicly
case 409: // Error allowed publicly
case 412: // Error allowed publicly
case 416: // Error allowed publicly
case 429: // Error allowed publicly
case 501: // Error allowed publicly
case 503: // Error allowed publicly

2
app/http.php
View file

@ -21,7 +21,7 @@ use Utopia\Logger\Log\User;
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
$payloadSize = max(4000000 /* 4mb */, App::getEnv('_APP_STORAGE_LIMIT', 10000000 /* 10mb */));
$payloadSize = 6 * (1024 * 1024); // 6MB
$http
->set([

3
app/init.php
View file

@ -78,6 +78,7 @@ const APP_STORAGE_FUNCTIONS = '/storage/functions';
const APP_STORAGE_CACHE = '/storage/cache';
const APP_STORAGE_CERTIFICATES = '/storage/certificates';
const APP_STORAGE_CONFIG = '/storage/config';
const APP_STORAGE_READ_BUFFER = 20 * (1024 * 1024); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT`
const APP_SOCIAL_TWITTER = 'https://twitter.com/appwrite';
const APP_SOCIAL_TWITTER_HANDLE = 'appwrite';
const APP_SOCIAL_FACEBOOK = 'https://www.facebook.com/appwrite.io';
@ -121,6 +122,8 @@ const APP_AUTH_TYPE_SESSION = 'Session';
const APP_AUTH_TYPE_JWT = 'JWT';
const APP_AUTH_TYPE_KEY = 'Key';
const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 2*1024*1024; // 2MB
$register = new Registry();

11
app/views/console/storage/bucket.phtml
View file

@ -158,7 +158,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
<input type="hidden" name="fileId" data-ls-bind="{{file.$id}}" />
</form>
</div>
<div class="col span-4 text-size-small">
<div data-ls-if="{{file.chunksTotal}} == {{file.chunksUploaded}}" class="col span-4 text-size-small">
<div class="margin-bottom-small">File Preview</div>
<div class="margin-bottom-small">
@ -188,7 +188,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
<footer>
<button class="link pull-end text-danger" data-ls-ui-trigger="file-delete-{{file.$id}},modal-close">Delete File</button>
<button type="button" data-ls-ui-trigger="file-update-{{file.$id}},modal-close">Update</button> &nbsp; <button data-ui-modal-close="" type="button" class="reverse desktops-only-inline tablets-only-inline">Cancel</button>
<button type="button" data-ls-if="{{file.chunksTotal}} == {{file.chunksUploaded}}" data-ls-ui-trigger="file-update-{{file.$id}},modal-close">Update</button> &nbsp; <button data-ui-modal-close="" type="button" class="reverse desktops-only-inline tablets-only-inline">Cancel</button>
</footer>
</div>
</td>
@ -196,8 +196,11 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
<span class="text-fade text-size-small" data-ls-bind="{{file.mimeType}}"></span>
</td>
<td data-title="Size: ">
<span class="text-fade text-size-small" data-ls-bind="{{file.sizeOriginal|humanFileSize}}"></span>
<span class="text-fade text-size-small" data-ls-bind="{{file.sizeOriginal|humanFileUnit}}"></span>
<div data-ls-if="{{file.chunksTotal}} == {{file.chunksUploaded}}" >
<span class="text-fade text-size-small" data-ls-bind="{{file.sizeOriginal|humanFileSize}}"></span>
<span class="text-fade text-size-small" data-ls-bind="{{file.sizeOriginal|humanFileUnit}}"></span>
</div>
<span class="text-fade text-size-small" data-ls-if="{{file.chunksTotal}} != {{file.chunksUploaded}}">incomplete</span>
</td>
<td data-title="Created: ">
<span class="text-fade text-size-small" data-ls-bind="{{file.dateCreated|dateText}}"></span>

33
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "ab493f0a7f01a1105f8bc5caaf9b928b",
"content-hash": "d9d66a3e700bc3936fe9a09f3c1b38f9",
"packages": [
{
"name": "adhocore/jwt",
@ -2141,16 +2141,16 @@
},
{
"name": "utopia-php/database",
"version": "0.14.0",
"version": "0.14.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "2f2527bb080cf578fba327ea2ec637064561d403"
"reference": "ecc143f2cfe16b23675407035c6b5375ba263285"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/2f2527bb080cf578fba327ea2ec637064561d403",
"reference": "2f2527bb080cf578fba327ea2ec637064561d403",
"url": "https://api.github.com/repos/utopia-php/database/zipball/ecc143f2cfe16b23675407035c6b5375ba263285",
"reference": "ecc143f2cfe16b23675407035c6b5375ba263285",
"shasum": ""
},
"require": {
@ -2198,9 +2198,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.14.0"
"source": "https://github.com/utopia-php/database/tree/0.14.1"
},
"time": "2022-01-21T16:34:34+00:00"
"time": "2022-01-25T13:01:20+00:00"
},
{
"name": "utopia-php/domains",
@ -3126,23 +3126,23 @@
},
{
"name": "composer/pcre",
"version": "1.0.0",
"version": "1.0.1",
"source": {
"type": "git",
"url": "https://github.com/composer/pcre.git",
"reference": "3d322d715c43a1ac36c7fe215fa59336265500f2"
"reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/composer/pcre/zipball/3d322d715c43a1ac36c7fe215fa59336265500f2",
"reference": "3d322d715c43a1ac36c7fe215fa59336265500f2",
"url": "https://api.github.com/repos/composer/pcre/zipball/67a32d7d6f9f560b726ab25a061b38ff3a80c560",
"reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560",
"shasum": ""
},
"require": {
"php": "^5.3.2 || ^7.0 || ^8.0"
},
"require-dev": {
"phpstan/phpstan": "^1",
"phpstan/phpstan": "^1.3",
"phpstan/phpstan-strict-rules": "^1.1",
"symfony/phpunit-bridge": "^4.2 || ^5"
},
@ -3177,7 +3177,7 @@
],
"support": {
"issues": "https://github.com/composer/pcre/issues",
"source": "https://github.com/composer/pcre/tree/1.0.0"
"source": "https://github.com/composer/pcre/tree/1.0.1"
},
"funding": [
{
@ -3193,7 +3193,7 @@
"type": "tidelift"
}
],
"time": "2021-12-06T15:17:27+00:00"
"time": "2022-01-21T20:24:37+00:00"
},
{
"name": "composer/semver",
@ -3697,6 +3697,9 @@
"require": {
"php": "^7.1 || ^8.0"
},
"replace": {
"myclabs/deep-copy": "self.version"
},
"require-dev": {
"doctrine/collections": "^1.0",
"doctrine/common": "^2.6",
@ -6662,5 +6665,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.2.0"
"plugin-api-version": "2.1.0"
}

8
docs/references/storage/create-file.md
View file

@ -1 +1,7 @@
Create a new file. The user who creates the file will automatically be assigned to read and write access unless he has passed custom values for read and write arguments.
Create a new file. The user that creates the file will automatically be granted read and write permissions unless they have passed custom values for read and write permissions.
Larger files should be uploaded using multiple requests with the [content-range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range) header to send a partial request with a maximum supported chunk of `5MB`. The `content-range` header values should always be in bytes.
When the first request is sent, the server will return the **File** object, and the subsequent part request must include the file's **id** in `x-appwrite-upload-id` header to allow the server to know that the partial upload is for the existing file and not for a new one.
If you're creating a new file using one the Appwrite SDKs, all the chunking logic will be managed by the SDK internally.

12
src/Appwrite/Utopia/Response/Model/File.php
View file

@ -66,6 +66,18 @@ class File extends Model
'default' => 0,
'example' => 17890,
])
->addRule('chunksTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks available',
'default' => 0,
'example' => 17890,
])
->addRule('chunksUploaded', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks uploaded',
'default' => 0,
'example' => 17890,
])
;
}

51
tests/e2e/Services/Functions/FunctionsCustomServerTest.php
View file

@ -292,6 +292,41 @@ class FunctionsCustomServerTest extends Scope
$this->assertIsInt($tag['body']['dateCreated']);
$this->assertEquals('php index.php', $tag['body']['command']);
$this->assertGreaterThan(10000, $tag['body']['size']);
/**
* Test for Large Code File SUCCESS
*/
$source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz');
$chunkSize = 5*1024*1024;
$handle = @fopen($source, "rb");
$mimeType = 'application/x-gzip';
$counter = 0;
$size = filesize($source);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id']
];
$id = '';
while (!feof($handle)) {
$curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz');
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
if(!empty($id)) {
$headers['x-appwrite-id'] = $id;
}
$largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [
'command' => 'php index.php',
'code' => $curlFile,
]);
$counter++;
$id = $largeTag['body']['$id'];
}
@fclose($handle);
$this->assertEquals(201, $largeTag['headers']['status-code']);
$this->assertNotEmpty($largeTag['body']['$id']);
$this->assertIsInt($largeTag['body']['dateCreated']);
$this->assertEquals('php index.php', $largeTag['body']['command']);
$this->assertGreaterThan(10000, $largeTag['body']['size']);
/**
* Test for FAILURE
@ -342,9 +377,9 @@ class FunctionsCustomServerTest extends Scope
], $this->getHeaders()));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
/**
* Test search queries
@ -357,9 +392,9 @@ class FunctionsCustomServerTest extends Scope
]));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
$this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']);
$function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([
@ -370,9 +405,9 @@ class FunctionsCustomServerTest extends Scope
]));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
$this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']);
$function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([
@ -383,9 +418,9 @@ class FunctionsCustomServerTest extends Scope
]));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
$this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']);
return $data;

101
tests/e2e/Services/Storage/StorageBase.php
View file

@ -65,24 +65,48 @@ trait StorageBase
]);
$this->assertEquals(201, $bucket2['headers']['status-code']);
$this->assertNotEmpty($bucket2['body']['$id']);
/**
* Chunked Upload
*/
$file2 = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge([
$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
$totalSize = \filesize($source);
$chunkSize = 5*1024*1024;
$handle = @fopen($source, "rb");
$fileId = 'unique()';
$mimeType = mime_content_type($source);
$counter = 0;
$size = filesize($source);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => 'unique()',
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4'), 'video/mp4', 'large-file.mp4'),
'read' => ['role:all'],
'write' => ['role:all'],
]);
$this->assertEquals(201, $file2['headers']['status-code']);
$this->assertNotEmpty($file2['body']['$id']);
$this->assertIsInt($file2['body']['dateCreated']);
$this->assertEquals('large-file.mp4', $file2['body']['name']);
$this->assertEquals('video/mp4', $file2['body']['mimeType']);
$this->assertEquals(23660615, $file2['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $file2['body']['signature']); // should validate that the file is not encrypted
'x-appwrite-project' => $this->getProject()['$id']
];
$id = '';
while (!feof($handle)) {
$curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'large-file.mp4');
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
if(!empty($id)) {
$headers['x-appwrite-id'] = $id;
}
$largeFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [
'fileId' => $fileId,
'file' => $curlFile,
'read' => ['role:all'],
'write' => ['role:all'],
]);
$counter++;
$id = $largeFile['body']['$id'];
}
@fclose($handle);
$this->assertEquals(201, $largeFile['headers']['status-code']);
$this->assertNotEmpty($largeFile['body']['$id']);
$this->assertIsInt($largeFile['body']['dateCreated']);
$this->assertEquals('large-file.mp4', $largeFile['body']['name']);
$this->assertEquals('video/mp4', $largeFile['body']['mimeType']);
$this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted
/**
* Test for FAILURE unknown Bucket
@ -133,7 +157,7 @@ trait StorageBase
$this->assertEquals(400, $res['headers']['status-code']);
$this->assertEquals('File extension not allowed', $res['body']['message']);
return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $file2['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']];
return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $largeFile['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']];
}
/**
@ -263,6 +287,49 @@ trait StorageBase
$this->assertEquals('image/png', $file5['headers']['content-type']);
$this->assertNotEmpty($file5['body']);
// Test ranged download
$file51 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=0-99',
], $this->getHeaders()));
$path = __DIR__ . '/../../../resources/logo.png';
$originalChunk = \file_get_contents($path, false, null, 0, 100);
$this->assertEquals(206, $file51['headers']['status-code']);
$this->assertEquals('attachment; filename="logo.png"', $file51['headers']['content-disposition']);
$this->assertEquals('image/png', $file51['headers']['content-type']);
$this->assertNotEmpty($file51['body']);
$this->assertEquals($originalChunk, $file51['body']);
// Test ranged download - with invalid range
$file52 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=0-',
], $this->getHeaders()));
$this->assertEquals(206, $file52['headers']['status-code']);
// Test ranged download - with invalid range
$file53 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=988',
], $this->getHeaders()));
$this->assertEquals(416, $file53['headers']['status-code']);
// Test ranged download - with invalid range
$file54 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=-988',
], $this->getHeaders()));
$this->assertEquals(416, $file54['headers']['status-code']);
$file6 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/view', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],

BIN
tests/resources/functions/php-large.tar.gz Normal file
View file

Binary file not shown.