Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-21 07:58:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #2419 from appwrite/feat-s3-integration

Browse source
This commit is contained in:
Damodar Lohani 2022-01-30 13:43:35 +05:45 committed by GitHub
commit ebaed8b6f7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 2212 additions and 1473 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.env
View file

@ -20,6 +20,7 @@ _APP_DB_PORT=3306
_APP_DB_SCHEMA=appwrite
_APP_DB_USER=user
_APP_DB_PASS=password
_APP_STORAGE_DEVICE=Local
_APP_STORAGE_ANTIVIRUS=disabled
_APP_STORAGE_ANTIVIRUS_HOST=clamav
_APP_STORAGE_ANTIVIRUS_PORT=3310

9
Dockerfile
View file

@ -152,6 +152,15 @@ ENV _APP_SERVER=swoole \
_APP_STORAGE_ANTIVIRUS=enabled \
_APP_STORAGE_ANTIVIRUS_HOST=clamav \
_APP_STORAGE_ANTIVIRUS_PORT=3310 \
_APP_STORAGE_DEVICE=Local \
_APP_STORAGE_DEVICE_S3_ACCESS_KEY= \
_APP_STORAGE_DEVICE_S3_SECRET= \
_APP_STORAGE_DEVICE_S3_REGION= \
_APP_STORAGE_DEVICE_S3_BUCKET= \
_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY= \
_APP_STORAGE_DEVICE_DO_SPACES_SECRET= \
_APP_STORAGE_DEVICE_DO_SPACES_REGION= \
_APP_STORAGE_DEVICE_DO_SPACES_BUCKET= \
_APP_REDIS_HOST=redis \
_APP_REDIS_PORT=6379 \
_APP_DB_HOST=mariadb \

241
app/config/collections.php
View file

@ -6,6 +6,15 @@ use Utopia\Database\Database;
$providers = Config::getParam('providers', []);
$auth = Config::getParam('auth', []);
/**
* $collection => id of the parent collection where this will be inserted
* $id => id of this collection
* name => name of this collection
* project => whether or not this collection should be created per project
* attributes => list of attributes
* indexes => list of indexes
*/
$collections = [
'collections' => [
'$collection' => Database::METADATA,
@ -2045,9 +2054,7 @@ $collections = [
'buckets' => [
'$collection' => Database::METADATA,
'$id' => 'buckets',
'$permissions' => ['read' => ['*']],
'name' => 'Buckets',
'structure' => true,
'attributes' => [
[
'$id' => 'dateCreated',
@ -2064,7 +2071,7 @@ $collections = [
'type' => Database::VAR_INTEGER,
'size' => 0,
'format' => '',
'signed' => true,
'signed' => false,
'required' => false,
'array' => false,
'filters' => [],
@ -2123,8 +2130,8 @@ $collections = [
[
'$id' => 'maximumFileSize',
'type' => Database::VAR_INTEGER,
'signed' => true,
'size' => 0,
'signed' => false,
'size' => 8,
'format' => '',
'filters' => [],
'required' => true,
@ -2151,7 +2158,7 @@ $collections = [
'array' => false,
],
[
'$id' => 'antiVirus',
'$id' => 'antivirus',
'type' => Database::VAR_BOOLEAN,
'signed' => true,
'size' => 0,
@ -2325,6 +2332,228 @@ $collections = [
],
]
],
'files' => [
'$collection' => 'buckets',
'$id' => 'files',
'$name' => 'Files',
'attributes' => [
[
'$id' => 'dateCreated',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'array' => false,
'$id' => 'bucketId',
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'name',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'path',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'mimeType',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'metadata',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['json'],
],
[
'$id' => 'sizeOriginal',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'sizeActual',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'algorithm',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'comment',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'openSSLVersion',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 64,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'openSSLCipher',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 64,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'openSSLTag',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'openSSLIV',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'chunksTotal',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'chunksUploaded',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'search',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_search',
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_bucket',
'type' => Database::INDEX_KEY,
'attributes' => ['bucketId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
]
],
];
return $collections;

1
app/config/specs/0.13.x.client.json
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/0.13.x.console.json
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/0.13.x.server.json
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/open-api3-0.13.x-client.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/open-api3-0.13.x-console.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/open-api3-0.13.x-server.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/swagger2-0.13.x-client.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/swagger2-0.13.x-console.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/swagger2-0.13.x-server.json Normal file
View file

File diff suppressed because one or more lines are too long

72
app/config/variables.php
View file

@ -420,6 +420,78 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_STORAGE_DEVICE',
'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\' and \'DoSpaces\'.',
'introduction' => '0.13.0',
'default' => 'Local',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_S3_ACCESS_KEY',
'description' => 'AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_S3_SECRET',
'description' => 'AWS S3 storage secret key. Required when the storage adapter is set to S3. You can get your secret key from your AWS console.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_S3_REGION',
'description' => 'AWS S3 storage region. Required when storage adapter is set to S3. You can find your region info for your bucket from AWS console.',
'introduction' => '0.13.0',
'default' => 'us-eas-1',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_S3_BUCKET',
'description' => 'AWS S3 storage bucket. Required when storage adapter is set to S3. You can create buckets in your AWS console.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY',
'description' => 'DigitalOcean spaces access key. Required when the storage adapter is set to DoSpaces. You can get your access key from your DigitalOcean console.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_DO_SPACES_SECRET',
'description' => 'DigitalOcean spaces secret key. Required when the storage adapter is set to DoSpaces. You can get your secret key from your DigitalOcean console.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_DO_SPACES_REGION',
'description' => 'DigitalOcean spaces region. Required when storage adapter is set to DoSpaces. You can find your region info for your space from DigitalOcean console.',
'introduction' => '0.13.0',
'default' => 'us-eas-1',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE_DO_SPACES_BUCKET',
'description' => 'DigitalOcean spaces bucket. Required when storage adapter is set to DoSpaces. You can create spaces in your DigitalOcean console.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
'question' => '',
],
],
],
[

45
app/controllers/api/functions.php
View file

@ -452,11 +452,15 @@ App::post('/v1/functions/:functionId/tags')
->inject('response')
->inject('dbForProject')
->inject('usage')
->action(function ($functionId, $command, $file, $request, $response, $dbForProject, $usage) {
/** @var Appwrite\Utopia\Request $request */
->inject('deviceFunctions')
->inject('deviceLocal')
->action(function ($functionId, $command, $file, $request, $response, $dbForProject, $usage, $deviceFunctions, $deviceLocal) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForProject */
/** @var Appwrite\Event\Event $usage */
/** @var Utopia\Storage\Device $deviceFunctions */
/** @var Utopia\Storage\Device $deviceLocal */
$function = $dbForProject->getDocument('functions', $functionId);
@ -465,9 +469,8 @@ App::post('/v1/functions/:functionId/tags')
}
$file = $request->getFiles('code');
$device = Storage::getDevice('functions');
$fileExt = new FileExt([FileExt::TYPE_GZIP]);
$fileSize = new FileSize(App::getEnv('_APP_STORAGE_LIMIT', 0));
$fileSizeValidator = new FileSize(App::getEnv('_APP_STORAGE_LIMIT', 0));
$upload = new Upload();
if (empty($file)) {
@ -477,7 +480,7 @@ App::post('/v1/functions/:functionId/tags')
// Make sure we handle a single file and multiple files the same way
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed
throw new Exception('File type not allowed', 400);
@ -491,23 +494,23 @@ App::post('/v1/functions/:functionId/tags')
if (!empty($contentRange)) {
$start = $request->getContentRangeStart();
$end = $request->getContentRangeEnd();
$size = $request->getContentRangeSize();
$fileSize = $request->getContentRangeSize();
$tagId = $request->getHeader('x-appwrite-id', $tagId);
if(is_null($start) || is_null($end) || is_null($size)) {
if(is_null($start) || is_null($end) || is_null($fileSize)) {
throw new Exception('Invalid content-range header', 400);
}
if ($end == $size) {
if ($end === $fileSize) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($size / ($end + 1 - $start));
$chunks = (int) ceil($fileSize / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start)) + 1;
}
}
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
if (!$fileSizeValidator->isValid($fileSize)) { // Check if file size is exceeding allowed limit
throw new Exception('File size not allowed', 400);
}
@ -516,8 +519,8 @@ App::post('/v1/functions/:functionId/tags')
}
// Save to storage
$size ??= $device->getFileSize($fileTmpName);
$path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION));
$fileSize ??= $deviceLocal->getFileSize($fileTmpName);
$path = $deviceFunctions->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION));
$tag = $dbForProject->getDocument('tags', $tagId);
@ -528,14 +531,14 @@ App::post('/v1/functions/:functionId/tags')
}
}
$chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
$chunksUploaded = $deviceFunctions->upload($fileTmpName, $path, $chunk, $chunks);
if (empty($chunksUploaded)) {
throw new Exception('Failed moving file', 500);
}
if($chunksUploaded == $chunks) {
$size = $device->getFileSize($path);
if($chunksUploaded === $chunks) {
$fileSize = $deviceFunctions->getFileSize($path);
if ($tag->isEmpty()) {
$tag = $dbForProject->createDocument('tags', new Document([
@ -546,11 +549,11 @@ App::post('/v1/functions/:functionId/tags')
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => $size,
'size' => $fileSize,
'search' => implode(' ', [$tagId, $command]),
]));
} else {
$tag = $dbForProject->updateDocument('tags', $tagId, $tag->setAttribute('size', $size));
$tag = $dbForProject->updateDocument('tags', $tagId, $tag->setAttribute('size', $fileSize));
}
} else {
if($tag->isEmpty()) {
@ -689,10 +692,12 @@ App::delete('/v1/functions/:functionId/tags/:tagId')
->inject('response')
->inject('dbForProject')
->inject('usage')
->action(function ($functionId, $tagId, $response, $dbForProject, $usage) {
->inject('deviceFunctions')
->action(function ($functionId, $tagId, $response, $dbForProject, $usage, $deviceFunctions) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForProject */
/** @var Appwrite\Event\Event $usage */
/** @var Utopia\Storage\Device $deviceFunctions */
$function = $dbForProject->getDocument('functions', $functionId);
@ -710,9 +715,7 @@ App::delete('/v1/functions/:functionId/tags/:tagId')
throw new Exception('Tag not found', 404);
}
$device = Storage::getDevice('functions');
if ($device->delete($tag->getAttribute('path', ''))) {
if ($deviceFunctions->delete($tag->getAttribute('path', ''))) {
if (!$dbForProject->deleteDocument('tags', $tag->getId())) {
throw new Exception('Failed to remove tag from DB', 500);
}

3
app/controllers/api/projects.php
View file

@ -114,6 +114,9 @@ App::post('/v1/projects')
$adapter->setup();
foreach ($collections as $key => $collection) {
if(($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
$attributes = [];
$indexes = [];

682
app/controllers/api/storage.php
View file

File diff suppressed because it is too large Load diff

4
app/controllers/general.php
View file

@ -196,7 +196,7 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons
->addHeader('Server', 'Appwrite')
->addHeader('X-Content-Type-Options', 'nosniff')
->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE')
->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, Cache-Control, Expires, Pragma')
->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, X-Appwrite-ID, Content-Range, Range, Cache-Control, Expires, Pragma')
->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies')
->addHeader('Access-Control-Allow-Origin', $refDomain)
->addHeader('Access-Control-Allow-Credentials', 'true')
@ -314,7 +314,7 @@ App::options(function ($request, $response) {
$response
->addHeader('Server', 'Appwrite')
->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE')
->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, Cache-Control, Expires, Pragma, X-Fallback-Cookies')
->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, X-Appwrite-ID, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies')
->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies')
->addHeader('Access-Control-Allow-Origin', $origin)
->addHeader('Access-Control-Allow-Credentials', 'true')

29
app/controllers/shared/api.php
View file

@ -8,7 +8,9 @@ use Utopia\Exception;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Database\Document;
use Utopia\Storage\Device\DoSpaces;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Storage;
App::init(function ($utopia, $request, $response, $project, $user, $events, $audits, $usage, $deletes, $database, $dbForProject, $mode) {
@ -26,8 +28,31 @@ App::init(function ($utopia, $request, $response, $project, $user, $events, $aud
/** @var Appwrite\Event\Event $functions */
/** @var Utopia\Database\Database $dbForProject */
Storage::setDevice('files', new Local(APP_STORAGE_UPLOADS.'/app-'.$project->getId()));
Storage::setDevice('functions', new Local(APP_STORAGE_FUNCTIONS.'/app-'.$project->getId()));
Storage::setDevice('self', new Local());
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:default:
Storage::setDevice('files', new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()));
Storage::setDevice('functions', new Local(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()));
break;
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
Storage::setDevice('files', new S3(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl));
Storage::setDevice('functions', new S3(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl));
break;
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
Storage::setDevice('files', new DoSpaces(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl));
Storage::setDevice('functions', new DoSpaces(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl));
break;
}
$route = $utopia->match($request);

61
app/http.php
View file

@ -13,6 +13,7 @@ use Utopia\Config\Config;
use Utopia\Database\Validator\Authorization;
use Utopia\Audit\Audit;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Swoole\Files;
use Appwrite\Utopia\Request;
@ -109,10 +110,12 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
}
foreach ($collections as $key => $collection) {
if(($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if(!$dbForConsole->getCollection($key)->isEmpty()) {
continue;
}
Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...');
$attributes = [];
@ -141,8 +144,62 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
}
$dbForConsole->createCollection($key, $attributes, $indexes);
}
if($dbForConsole->getDocument('buckets', 'default')->isEmpty()) {
Console::success('[Setup] - Creating default bucket...');
$dbForConsole->createDocument('buckets', new Document([
'$id' => 'default',
'$collection' => 'buckets',
'dateCreated' => \time(),
'dateUpdated' => \time(),
'name' => 'Default',
'permission' => 'file',
'maximumFileSize' => (int) App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB
'allowedFileExtensions' => [],
'enabled' => true,
'adapter' => '',
'encryption' => true,
'antivirus' => true,
'$read' => ['role:all'],
'$write' => ['role:all'],
'search' => 'buckets Default',
]));
Console::success('[Setup] - Creating files collection for default bucket...');
$files = $collections['files'] ?? [];
if(empty($files)) {
throw new Exception('Files collection is not configured.');
}
$attributes = [];
$indexes = [];
foreach ($files['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => $attribute['$id'],
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
]);
}
foreach ($files['indexes'] as $index) {
$indexes[] = new Document([
'$id' => $index['$id'],
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$dbForConsole->createCollection('bucket_' . 'default', $attributes, $indexes);
}
Console::success('[Setup] - Server database init completed...');
});

51
app/init.php
View file

@ -50,6 +50,10 @@ use Swoole\Database\PDOPool;
use Swoole\Database\RedisConfig;
use Swoole\Database\RedisPool;
use Utopia\Database\Query;
use Utopia\Storage\Storage;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Device\DoSpaces;
const APP_NAME = 'Appwrite';
const APP_DOMAIN = 'appwrite.io';
@ -825,6 +829,53 @@ App::setResource('dbForConsole', function($db, $cache) {
return $database;
}, ['db', 'cache']);
App::setResource('deviceLocal', function() {
return new Local();
});
App::setResource('deviceFiles', function($project) {
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:default:
return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DoSpaces(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
}
}, ['project']);
App::setResource('deviceFunctions', function($project) {
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:default:
return new Local(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId());
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DoSpaces(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
}
}, ['project']);
App::setResource('mode', function($request) {
/** @var Appwrite\Utopia\Request $request */

22
app/tasks/usage.php
View file

@ -404,7 +404,7 @@ $cli
// Get total storage
$dbForProject->setNamespace('_project_' . $projectId);
$storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size');
$storageTotal = $dbForProject->sum('tags', 'size');
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_storage.tags.total'); //Construct unique id for each metric using time, period and metric
@ -427,7 +427,7 @@ $cli
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric
$id = \md5($time . '_1d_storage.tags.total'); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
@ -448,23 +448,23 @@ $cli
$collections = [
'users' => [
'namespace' => 'internal',
'namespace' => '',
],
'collections' => [
'metricPrefix' => 'database',
'namespace' => 'internal',
'namespace' => '',
'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
'documents' => [
'namespace' => 'external',
'namespace' => '',
],
],
],
'buckets' => [
'metricPrefix' => 'storage',
'namespace' => 'internal',
'namespace' => '',
'subCollections' => [
'files' => [
'namespace' => 'external',
'namespace' => '',
'collectionPrefix' => 'bucket_',
'sum' => [
'field' => 'sizeOriginal'
@ -543,7 +543,7 @@ $cli
foreach ($parents as $parent) {
foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
$dbForProject->setNamespace("_project_{$projectId}_{$subOptions['namespace']}");
$dbForProject->setNamespace("_project_{$projectId}");
$count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getId());
$subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
@ -597,12 +597,12 @@ $cli
continue;
}
$dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}");
$dbForProject->setNamespace("_project_{$projectId}");
$total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getId(), $sum['field']);
$subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total
$dbForProject->setNamespace("project_{$projectId}_internal");
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.total";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
@ -696,7 +696,7 @@ $cli
* Inserting project level sums for sub collections like storage.total
*/
foreach ($subCollectionTotals as $subCollection => $count) {
$dbForProject->setNamespace("project_{$projectId}_internal");
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total";

10
app/views/console/storage/bucket.phtml
View file

@ -216,6 +216,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
data-service="storage.listFiles"
data-event="submit"
data-param-search="{{router.params.search}}"
data-param-bucket-id="{{router.params.id}}"
data-param-limit="<?php echo APP_PAGING_LIMIT; ?>"
data-param-order-type="DESC"
data-scope="sdk"
@ -230,6 +231,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
<form
data-service="storage.listFiles"
data-event="submit"
data-param-bucket-id="{{router.params.id}}"
data-param-search="{{router.params.search}}"
data-param-limit="<?php echo APP_PAGING_LIMIT; ?>"
data-param-order-type="DESC"
@ -406,7 +408,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
</div>
<div class="margin-bottom">
<input name="antiVirus" type="hidden" data-forms-switch data-cast-to="boolean" data-ls-bind="{{project-bucket.antiVirus}}" /> &nbsp; Anti Virus <span class="tooltip" data-tooltip="Mark whether anti virus scanning is enabled"><i class="icon-info-circled"></i></span>
<input name="antivirus" type="hidden" data-forms-switch data-cast-to="boolean" data-ls-bind="{{project-bucket.antivirus}}" /> &nbsp; Anti Virus <span class="tooltip" data-tooltip="Mark whether anti virus scanning is enabled"><i class="icon-info-circled"></i></span>
</div>
<label for="bucket-allowedFileExtensions">Allowed File Extensions</label>
@ -436,14 +438,14 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
<p class="text-fade margin-top-tiny">With Bucket Level permissions, you assign permissions only once in the bucket.</p>
<p class="text-fade margin-top-tiny">In this permission level permissions assigned to bucket takes the precedence and file permissions are ignored</p>
<div data-ls-if="{{project-bucket.permission}} == 'bucket'">
<label for="bucket-read">Read Access <span class="text-size-small">(<a data-ls-attrs="href={{env.HOME}}/docs/permissions" target="_blank" rel="noopener">Learn more</a>)</span></label>
<input type="hidden" id="bucket-read" name="read" data-forms-tags data-cast-to="json" data-ls-bind="{{project-bucket.$permissions.read}}" placeholder="User ID, Team ID or Role" />
<div class="text-fade text-size-xs margin-top-negative-small margin-bottom">Add * for wildcard access</div>
<div class="text-fade text-size-xs margin-top-negative-small margin-bottom">Add 'role:all' for wildcard access</div>
<label for="bucket-write">Write Access <span class="text-size-small">(<a data-ls-attrs="href={{env.HOME}}/docs/permissions" target="_blank" rel="noopener">Learn more</a>)</label>
<input type="hidden" id="bucket-write" name="write" data-forms-tags data-cast-to="json" data-ls-bind="{{project-bucket.$permissions.write}}" placeholder="User ID, Team ID or Role" />
<div class="text-fade text-size-xs margin-top-negative-small margin-bottom">Add * for wildcard access</div>
<div class="text-fade text-size-xs margin-top-negative-small margin-bottom">Add 'role:all' for wildcard access</div>
</div>
</div>
</div>

32
app/workers/deletes.php
View file

@ -1,11 +1,15 @@
<?php
use Utopia\App;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Appwrite\Resque\Worker;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Device\DoSpaces;
use Utopia\Storage\Storage;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\CLI\Console;
@ -455,12 +459,30 @@ class DeletesV1 extends Worker
protected function deleteBucket(Document $document, string $projectId)
{
$bucketId = $document->getId();
$this->deleteByGroup($bucketId . '_files',[
new Query('bucketId', Query::TYPE_EQUAL, [$bucketId])
], $this->getExternalDB($projectId));
$dbForProject = $this->getProjectDB($projectId);
$dbForProject->deleteCollection('bucket_' . $bucketId);
$device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId);
$device->deletePath($device->getRoot() . DIRECTORY_SEPARATOR . $bucketId);
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
$device = new S3(APP_STORAGE_UPLOADS . '/app-' . $projectId, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
break;
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
$device = new DoSpaces(APP_STORAGE_UPLOADS . '/app-' . $projectId, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
break;
}
$device->deletePath($bucketId);
}
}

18
docker-compose.yml
View file

@ -112,6 +112,15 @@ services:
- _APP_STORAGE_ANTIVIRUS
- _APP_STORAGE_ANTIVIRUS_HOST
- _APP_STORAGE_ANTIVIRUS_PORT
- _APP_STORAGE_DEVICE
- _APP_STORAGE_DEVICE_S3_ACCESS_KEY
- _APP_STORAGE_DEVICE_S3_SECRET
- _APP_STORAGE_DEVICE_S3_REGION
- _APP_STORAGE_DEVICE_S3_BUCKET
- _APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DEVICE_DO_SPACES_SECRET
- _APP_STORAGE_DEVICE_DO_SPACES_REGION
- _APP_STORAGE_DEVICE_DO_SPACES_BUCKET
- _APP_SMTP_HOST
- _APP_SMTP_PORT
- _APP_SMTP_SECURE
@ -254,6 +263,15 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_STORAGE_DEVICE
- _APP_STORAGE_DEVICE_S3_ACCESS_KEY
- _APP_STORAGE_DEVICE_S3_SECRET
- _APP_STORAGE_DEVICE_S3_REGION
- _APP_STORAGE_DEVICE_S3_BUCKET
- _APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DEVICE_DO_SPACES_SECRET
- _APP_STORAGE_DEVICE_DO_SPACES_REGION
- _APP_STORAGE_DEVICE_DO_SPACES_BUCKET
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG

6
docs/references/storage/create-file.md
View file

@ -1,7 +1,7 @@
Create a new file. The user that creates the file will automatically be granted read and write permissions unless they have passed custom values for read and write permissions.
Create a new file. Before using this route, you should create a new bucket resource using either a [server integration](/docs/server/database#storageCreateBucket) API or directly from your Appwrite console.
Larger files should be uploaded using multiple requests with the [content-range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range) header to send a partial request with a maximum supported chunk of `5MB`. The `content-range` header values should always be in bytes.
When the first request is sent, the server will return the **File** object, and the subsequent part request must include the file's **id** in `x-appwrite-upload-id` header to allow the server to know that the partial upload is for the existing file and not for a new one.
When the first request is sent, the server will return the **File** object, and the subsequent part request must include the file's **id** in `x-appwrite-id` header to allow the server to know that the partial upload is for the existing file and not for a new one.
If you're creating a new file using one the Appwrite SDKs, all the chunking logic will be managed by the SDK internally.
If you're creating a new file using one of the Appwrite SDKs, all the chunking logic will be managed by the SDK internally.

1020
public/dist/scripts/app-all.js vendored
View file

File diff suppressed because one or more lines are too long

1014
public/dist/scripts/app-dep.js vendored
View file

File diff suppressed because it is too large Load diff

6
public/dist/scripts/app.js vendored
View file

File diff suppressed because one or more lines are too long

189
public/scripts/dependencies/appwrite.js
View file

@ -413,8 +413,9 @@
/**
* Update Account Preferences
*
* Update currently logged in user account preferences. You can pass only the
* specific settings you wish to update.
* Update currently logged in user account preferences. The object you pass is
* stored as is, and replaces any previous value. The maximum allowed prefs
* size is 64kB and throws error if exceeded.
*
* @param {object} prefs
* @throws {AppwriteException}
@ -1601,7 +1602,7 @@
/**
* Create String Attribute
*
* Create a new string attribute.
* Create a string attribute.
*
*
* @param {string} collectionId
@ -2534,7 +2535,7 @@
* @throws {AppwriteException}
* @returns {Promise}
*/
createTag: (functionId, command, code) => __awaiter(this, void 0, void 0, function* () {
createTag: (functionId, command, code, onProgress = (progress) => { }) => __awaiter(this, void 0, void 0, function* () {
if (typeof functionId === 'undefined') {
throw new AppwriteException('Missing required parameter: "functionId"');
}
@ -2553,9 +2554,38 @@
payload['code'] = code;
}
const uri = new URL(this.config.endpoint + path);
return yield this.call('post', uri, {
'content-type': 'multipart/form-data',
}, payload);
const size = code.size;
if (size <= Appwrite.CHUNK_SIZE) {
return yield this.call('post', uri, {
'content-type': 'multipart/form-data',
}, payload);
}
else {
let id = undefined;
let response = undefined;
const totalCounters = Math.ceil(size / Appwrite.CHUNK_SIZE);
for (let counter = 0; counter < totalCounters; counter++) {
const start = (counter * Appwrite.CHUNK_SIZE);
const end = Math.min((((counter * Appwrite.CHUNK_SIZE) + Appwrite.CHUNK_SIZE) - 1), size);
const headers = {
'content-type': 'multipart/form-data',
'content-range': 'bytes ' + start + '-' + end + '/' + size
};
if (id) {
headers['x-appwrite-id'] = id;
}
const stream = code.slice(start, end + 1);
payload['code'] = new File([stream], code.name);
response = yield this.call('post', uri, headers, payload);
if (!id) {
id = response['$id'];
}
if (onProgress) {
onProgress(Math.min((counter + 1) * Appwrite.CHUNK_SIZE, size) / size * 100);
}
}
return response;
}
}),
/**
* Get Tag
@ -3940,18 +3970,18 @@
* @param {string} bucketId
* @param {string} name
* @param {string} permission
* @param {string} read
* @param {string} write
* @param {string[]} read
* @param {string[]} write
* @param {number} maximumFileSize
* @param {string[]} allowedFileExtensions
* @param {boolean} enabled
* @param {string} adapter
* @param {boolean} encryption
* @param {boolean} antiVirus
* @param {boolean} antivirus
* @throws {AppwriteException}
* @returns {Promise}
*/
createBucket: (bucketId, name, permission, read, write, maximumFileSize, allowedFileExtensions, enabled, adapter, encryption, antiVirus) => __awaiter(this, void 0, void 0, function* () {
createBucket: (bucketId, name, permission, read, write, maximumFileSize, allowedFileExtensions, enabled, adapter, encryption, antivirus) => __awaiter(this, void 0, void 0, function* () {
if (typeof bucketId === 'undefined') {
throw new AppwriteException('Missing required parameter: "bucketId"');
}
@ -3993,8 +4023,8 @@
if (typeof encryption !== 'undefined') {
payload['encryption'] = encryption;
}
if (typeof antiVirus !== 'undefined') {
payload['antiVirus'] = antiVirus;
if (typeof antivirus !== 'undefined') {
payload['antivirus'] = antivirus;
}
const uri = new URL(this.config.endpoint + path);
return yield this.call('post', uri, {
@ -4029,28 +4059,35 @@
*
* @param {string} bucketId
* @param {string} name
* @param {string} read
* @param {string} write
* @param {string} permission
* @param {string[]} read
* @param {string[]} write
* @param {number} maximumFileSize
* @param {string[]} allowedFileExtensions
* @param {boolean} enabled
* @param {boolean} encryption
* @param {boolean} antiVirus
* @param {boolean} antivirus
* @throws {AppwriteException}
* @returns {Promise}
*/
updateBucket: (bucketId, name, read, write, maximumFileSize, allowedFileExtensions, enabled, encryption, antiVirus) => __awaiter(this, void 0, void 0, function* () {
updateBucket: (bucketId, name, permission, read, write, maximumFileSize, allowedFileExtensions, enabled, encryption, antivirus) => __awaiter(this, void 0, void 0, function* () {
if (typeof bucketId === 'undefined') {
throw new AppwriteException('Missing required parameter: "bucketId"');
}
if (typeof name === 'undefined') {
throw new AppwriteException('Missing required parameter: "name"');
}
if (typeof permission === 'undefined') {
throw new AppwriteException('Missing required parameter: "permission"');
}
let path = '/storage/buckets/{bucketId}'.replace('{bucketId}', bucketId);
let payload = {};
if (typeof name !== 'undefined') {
payload['name'] = name;
}
if (typeof permission !== 'undefined') {
payload['permission'] = permission;
}
if (typeof read !== 'undefined') {
payload['read'] = read;
}
@ -4069,8 +4106,8 @@
if (typeof encryption !== 'undefined') {
payload['encryption'] = encryption;
}
if (typeof antiVirus !== 'undefined') {
payload['antiVirus'] = antiVirus;
if (typeof antivirus !== 'undefined') {
payload['antivirus'] = antivirus;
}
const uri = new URL(this.config.endpoint + path);
return yield this.call('put', uri, {
@ -4146,19 +4183,34 @@
/**
* Create File
*
* Create a new file. The user who creates the file will automatically be
* assigned to read and write access unless he has passed custom values for
* read and write arguments.
* Create a new file. Before using this route, you should create a new bucket
* resource using either a [server
* integration](/docs/server/database#storageCreateBucket) API or directly
* from your Appwrite console.
*
* Larger files should be uploaded using multiple requests with the
* [content-range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range)
* header to send a partial request with a maximum supported chunk of `5MB`.
* The `content-range` header values should always be in bytes.
*
* When the first request is sent, the server will return the **File** object,
* and the subsequent part request must include the file's **id** in
* `x-appwrite-id` header to allow the server to know that the partial upload
* is for the existing file and not for a new one.
*
* If you're creating a new file using one of the Appwrite SDKs, all the
* chunking logic will be managed by the SDK internally.
*
*
* @param {string} bucketId
* @param {string} fileId
* @param {File} file
* @param {string} read
* @param {string} write
* @param {string[]} read
* @param {string[]} write
* @throws {AppwriteException}
* @returns {Promise}
*/
createFile: (bucketId, fileId, file, read, write) => __awaiter(this, void 0, void 0, function* () {
createFile: (bucketId, fileId, file, read, write, onProgress = (progress) => { }) => __awaiter(this, void 0, void 0, function* () {
if (typeof bucketId === 'undefined') {
throw new AppwriteException('Missing required parameter: "bucketId"');
}
@ -4183,9 +4235,38 @@
payload['write'] = write;
}
const uri = new URL(this.config.endpoint + path);
return yield this.call('post', uri, {
'content-type': 'multipart/form-data',
}, payload);
const size = file.size;
if (size <= Appwrite.CHUNK_SIZE) {
return yield this.call('post', uri, {
'content-type': 'multipart/form-data',
}, payload);
}
else {
let id = undefined;
let response = undefined;
const totalCounters = Math.ceil(size / Appwrite.CHUNK_SIZE);
for (let counter = 0; counter < totalCounters; counter++) {
const start = (counter * Appwrite.CHUNK_SIZE);
const end = Math.min((((counter * Appwrite.CHUNK_SIZE) + Appwrite.CHUNK_SIZE) - 1), size);
const headers = {
'content-type': 'multipart/form-data',
'content-range': 'bytes ' + start + '-' + end + '/' + size
};
if (id) {
headers['x-appwrite-id'] = id;
}
const stream = file.slice(start, end + 1);
payload['file'] = new File([stream], file.name);
response = yield this.call('post', uri, headers, payload);
if (!id) {
id = response['$id'];
}
if (onProgress) {
onProgress(Math.min((counter + 1) * Appwrite.CHUNK_SIZE, size) / size * 100);
}
}
return response;
}
}),
/**
* Get File
@ -4220,8 +4301,8 @@
*
* @param {string} bucketId
* @param {string} fileId
* @param {string} read
* @param {string} write
* @param {string[]} read
* @param {string[]} write
* @throws {AppwriteException}
* @returns {Promise}
*/
@ -4232,12 +4313,6 @@
if (typeof fileId === 'undefined') {
throw new AppwriteException('Missing required parameter: "fileId"');
}
if (typeof read === 'undefined') {
throw new AppwriteException('Missing required parameter: "read"');
}
if (typeof write === 'undefined') {
throw new AppwriteException('Missing required parameter: "write"');
}
let path = '/storage/buckets/{bucketId}/files/{fileId}'.replace('{bucketId}', bucketId).replace('{fileId}', fileId);
let payload = {};
if (typeof read !== 'undefined') {
@ -4326,9 +4401,9 @@
* @param {string} background
* @param {string} output
* @throws {AppwriteException}
* @returns {URL}
* @returns {Promise}
*/
getFilePreview: (bucketId, fileId, width, height, gravity, quality, borderWidth, borderColor, borderRadius, opacity, rotation, background, output) => {
getFilePreview: (bucketId, fileId, width, height, gravity, quality, borderWidth, borderColor, borderRadius, opacity, rotation, background, output) => __awaiter(this, void 0, void 0, function* () {
if (typeof bucketId === 'undefined') {
throw new AppwriteException('Missing required parameter: "bucketId"');
}
@ -4371,12 +4446,10 @@
payload['output'] = output;
}
const uri = new URL(this.config.endpoint + path);
payload['project'] = this.config.project;
for (const [key, value] of Object.entries(this.flatten(payload))) {
uri.searchParams.append(key, value);
}
return uri;
},
return yield this.call('get', uri, {
'content-type': 'application/json',
}, payload);
}),
/**
* Get File for View
*
@ -4793,6 +4866,10 @@
* after being redirected back to your app from the invitation email received
* by the user.
*
* If the request is successful, a session for the user is automatically
* created.
*
*
* @param {string} teamId
* @param {string} membershipId
* @param {string} userId
@ -5105,8 +5182,9 @@
/**
* Update User Preferences
*
* Update the user preferences by its unique ID. You can pass only the
* specific settings you wish to update.
* Update the user preferences by its unique ID. The object you pass is stored
* as is, and replaces any previous value. The maximum allowed prefs size is
* 64kB and throws error if exceeded.
*
* @param {string} userId
* @param {object} prefs
@ -5465,8 +5543,27 @@
return output;
}
}
Appwrite.CHUNK_SIZE = 5 * 1024 * 1024;
class Query {
}
Query.equal = (attribute, value) => Query.addQuery(attribute, "equal", value);
Query.notEqual = (attribute, value) => Query.addQuery(attribute, "notEqual", value);
Query.lesser = (attribute, value) => Query.addQuery(attribute, "lesser", value);
Query.lesserEqual = (attribute, value) => Query.addQuery(attribute, "lesserEqual", value);
Query.greater = (attribute, value) => Query.addQuery(attribute, "greater", value);
Query.greaterEqual = (attribute, value) => Query.addQuery(attribute, "greaterEqual", value);
Query.search = (attribute, value) => Query.addQuery(attribute, "search", value);
Query.addQuery = (attribute, oper, value) => value instanceof Array
? `${attribute}.${oper}(${value
.map((v) => Query.parseValues(v))
.join(",")})`
: `${attribute}.${oper}(${Query.parseValues(value)})`;
Query.parseValues = (value) => typeof value === "string" || value instanceof String
? `"${value}"`
: `${value}`;
exports.Appwrite = Appwrite;
exports.Query = Query;
Object.defineProperty(exports, '__esModule', { value: true });

2
public/scripts/views/forms/upload.js
View file

@ -108,7 +108,7 @@
expression.parse(element.dataset["write"] || "[]")
);
sdk.storage.createFile('unique()', files[0], read, write, 1).then(
sdk.storage.createFile('default', 'unique()', files[0], read, write, 1).then(
function(response) {
onComplete(message);

2
src/Appwrite/Utopia/Response/Model/Bucket.php
View file

@ -92,7 +92,7 @@ class Bucket extends Model
'default' => true,
'example' => false,
])
->addRule('antiVirus', [
->addRule('antivirus', [
'type' => self::TYPE_BOOLEAN,
'description' => 'Virus scanning is enabled.',
'default' => true,

11
test.http Normal file
View file

@ -0,0 +1,11 @@
POST http://localhost/v1/functions
Content-Type: application/json
x-appwrite-project:test
x-appwrite-key:c4b173b6623ba33c2457f4030218c3135f20e9ab4cd096efd3fb2b5eef72dff7ae3f41627dc141f6b5d405ab44eb7d9174b516a5678401fcb0a60b92888cecb7fe614a7b2cb7385863b29c02d8312d7cf823c731ae4bb385a9c21b6b9d2e5a245f979f1ebba1b09e8d0b0d66ef940a141789926b226abe4821bd191c087d6870
{
"functionId":"test1",
"name": "test1",
"runtime": "node-16.0",
"vars": {}
}

2
tests/e2e/General/HTTPTest.php
View file

@ -26,7 +26,7 @@ class HTTPTest extends Scope
$this->assertEquals(204, $response['headers']['status-code']);
$this->assertEquals('Appwrite', $response['headers']['server']);
$this->assertEquals('GET, POST, PUT, PATCH, DELETE', $response['headers']['access-control-allow-methods']);
$this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']);
$this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, X-Appwrite-ID, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']);
$this->assertEquals('X-Fallback-Cookies', $response['headers']['access-control-expose-headers']);
$this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
$this->assertEquals('true', $response['headers']['access-control-allow-credentials']);

117
tests/e2e/Services/Storage/StorageBase.php
View file

@ -107,6 +107,36 @@ trait StorageBase
$this->assertEquals('video/mp4', $largeFile['body']['mimeType']);
$this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted
/**
* Failure
* Test for Chunk above 5MB
*/
$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
$totalSize = \filesize($source);
$chunkSize = 6*1024*1024;
$handle = @fopen($source, "rb");
$fileId = 'unique()';
$mimeType = mime_content_type($source);
$counter = 0;
$size = filesize($source);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id']
];
$id = '';
$curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'large-file.mp4');
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
$res = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', $this->getHeaders(), [
'fileId' => $fileId,
'file' => $curlFile,
'read' => ['role:all'],
'write' => ['role:all'],
]);
@fclose($handle);
$this->assertEquals(413, $res['headers']['status-code']);
/**
* Test for FAILURE unknown Bucket
@ -394,6 +424,93 @@ trait StorageBase
return $data;
}
/**
* @depends testCreateBucketFile
*/
public function testFilePreviewCache(array $data): array
{
$bucketId = $data['bucketId'];
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => 'testcache',
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'logo.png'),
'read' => ['role:all'],
'write' => ['role:all'],
]);
$this->assertEquals(201, $file['headers']['status-code']);
$this->assertNotEmpty($file['body']['$id']);
$fileId = $file['body']['$id'];
//get image preview
$file3 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/preview', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'width' => 300,
'height' => 100,
'borderRadius' => '50',
'opacity' => '0.5',
'output' => 'png',
'rotation' => '45',
]);
$this->assertEquals(200, $file3['headers']['status-code']);
$this->assertEquals('image/png', $file3['headers']['content-type']);
$this->assertNotEmpty($file3['body']);
$imageBefore = new \Imagick();
$imageBefore->readImageBlob($file3['body']);
$file = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $data['bucketId'] . '/files/' . $fileId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(204, $file['headers']['status-code']);
$this->assertEmpty($file['body']);
//upload again using the same ID
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => 'testcache',
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/disk-b/kitten-2.png'), 'image/png', 'logo.png'),
'read' => ['role:all'],
'write' => ['role:all'],
]);
$this->assertEquals(201, $file['headers']['status-code']);
$this->assertNotEmpty($file['body']['$id']);
//get image preview after
$file3 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/preview', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'width' => 300,
'height' => 100,
'borderRadius' => '50',
'opacity' => '0.5',
'output' => 'png',
'rotation' => '45',
]);
$this->assertEquals(200, $file3['headers']['status-code']);
$this->assertEquals('image/png', $file3['headers']['content-type']);
$this->assertNotEmpty($file3['body']);
$imageAfter = new \Imagick();
$imageAfter->readImageBlob($file3['body']);
$this->assertNotEquals($imageBefore->getImageBlob(), $imageAfter->getImageBlob());
return $data;
}
/**
* @depends testCreateBucketFile
*/

25
tests/e2e/Services/Storage/StorageCustomServerTest.php
View file

@ -2,10 +2,10 @@
namespace Tests\E2E\Services\Storage;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideServer;
use Tests\E2E\Client;
class StorageCustomServerTest extends Scope
{
@ -13,7 +13,7 @@ class StorageCustomServerTest extends Scope
use ProjectCustom;
use SideServer;
public function testCreateBucket():array
public function testCreateBucket(): array
{
/**
* Test for SUCCESS
@ -35,7 +35,7 @@ class StorageCustomServerTest extends Scope
$this->assertEquals('Test Bucket', $bucket['body']['name']);
$this->assertEquals(true, $bucket['body']['enabled']);
$this->assertEquals(true, $bucket['body']['encryption']);
$this->assertEquals(true, $bucket['body']['antiVirus']);
$this->assertEquals(true, $bucket['body']['antivirus']);
$this->assertEquals('local', $bucket['body']['adapter']);
$bucketId = $bucket['body']['$id'];
@ -52,7 +52,7 @@ class StorageCustomServerTest extends Scope
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$this->assertEquals('bucket1', $bucket['body']['$id']);
/**
* Test for FAILURE
*/
@ -92,7 +92,7 @@ class StorageCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'cursor' => $response['body']['buckets'][0]['$id']
'cursor' => $response['body']['buckets'][0]['$id'],
]);
$this->assertEquals(200, $response['headers']['status-code']);
@ -122,18 +122,18 @@ class StorageCustomServerTest extends Scope
$this->assertNotEmpty($response['body']);
$this->assertEquals($id, $response['body']['$id']);
$this->assertEquals('Test Bucket', $response['body']['name']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/storage/buckets/empty',
array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(404, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/storage/buckets/id-is-really-long-id-is-really-long-id-is-really-long-id-is-really-long',
array_merge([
'content-type' => 'application/json',
@ -147,7 +147,7 @@ class StorageCustomServerTest extends Scope
/**
* @depends testCreateBucket
*/
public function testUpdateBucket(array $data):array
public function testUpdateBucket(array $data): array
{
$id = $data['bucketId'] ?? '';
/**
@ -160,6 +160,7 @@ class StorageCustomServerTest extends Scope
'bucketId' => 'unique()',
'name' => 'Test Bucket Updated',
'enabled' => false,
'permission' => 'file',
]);
$this->assertEquals(200, $bucket['headers']['status-code']);
$this->assertNotEmpty($bucket['body']['$id']);
@ -170,7 +171,7 @@ class StorageCustomServerTest extends Scope
$this->assertEquals('Test Bucket Updated', $bucket['body']['name']);
$this->assertEquals(false, $bucket['body']['enabled']);
$this->assertEquals(true, $bucket['body']['encryption']);
$this->assertEquals(true, $bucket['body']['antiVirus']);
$this->assertEquals(true, $bucket['body']['antivirus']);
$this->assertEquals('local', $bucket['body']['adapter']);
$bucketId = $bucket['body']['$id'];
/**
@ -204,7 +205,7 @@ class StorageCustomServerTest extends Scope
], $this->getHeaders()));
$this->assertEquals(204, $response['headers']['status-code']);
$this->assertEmpty($response['body']);
$response = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $id,
array_merge([
'content-type' => 'application/json',
@ -215,4 +216,4 @@ class StorageCustomServerTest extends Scope
return $data;
}
}
}

2
tests/e2e/Services/Webhooks/WebhooksBase.php
View file

@ -317,6 +317,7 @@ trait WebhooksBase
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'name' => 'Test Bucket Updated',
'permission' => 'file',
'enabled' => false,
]);
@ -354,6 +355,7 @@ trait WebhooksBase
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'name' => 'Test Bucket Updated',
'permission' => 'file',
'enabled' => true,
]);