From 46c3da70a979441152d589d0f226e95bf761832a Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Fri, 18 Mar 2022 16:03:04 +0000 Subject: [PATCH 01/17] add support for linode --- CHANGES.md | 5 +++++ Dockerfile | 4 ++++ app/config/variables.php | 32 ++++++++++++++++++++++++++++++++ app/executor.php | 8 ++++++++ app/init.php | 8 ++++++++ app/views/install/compose.phtml | 12 ++++++++++++ app/workers/deletes.php | 8 ++++++++ composer.lock | 14 +++++++------- docker-compose.yml | 12 ++++++++++++ src/Appwrite/Resque/Worker.php | 8 ++++++++ 10 files changed, 104 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2ac2f910bc..7b09391f49 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +# Latest +## Features +### Storage +- Support for Linode object storage + # Version 0.13.3 ## Bugs - Fixed search for terms that inlcude `@` characters diff --git a/Dockerfile b/Dockerfile index 023e88a205..204490f4d0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -162,6 +162,10 @@ ENV _APP_SERVER=swoole \ _APP_STORAGE_DO_SPACES_SECRET= \ _APP_STORAGE_DO_SPACES_REGION= \ _APP_STORAGE_DO_SPACES_BUCKET= \ + _APP_STORAGE_LINODE_ACCESS_KEY= \ + _APP_STORAGE_LINODE_SECRET= \ + _APP_STORAGE_LINODE_REGION= \ + _APP_STORAGE_LINODE_BUCKET= \ _APP_REDIS_HOST=redis \ _APP_REDIS_PORT=6379 \ _APP_DB_HOST=mariadb \ diff --git a/app/config/variables.php b/app/config/variables.php index 5cae4aa564..4d0f2ae117 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -501,6 +501,38 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_STORAGE_LINODE_ACCESS_KEY', + 'description' => 'Linode object storage access key. Required when the storage adapter is set to Linode. You can get your access key from your Linode console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_LINODE_SECRET', + 'description' => 'Linode object storage secret key. Required when the storage adapter is set to Linode. You can get your secret key from your Linode console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_LINODE_REGION', + 'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info for your space from Linode console.', + 'introduction' => '0.13.0', + 'default' => 'us-eas-1', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_LINODE_BUCKET', + 'description' => 'Linode object storage bucket. Required when storage adapter is set to Linode. You can create buckets in your Linode console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], ], ], [ diff --git a/app/executor.php b/app/executor.php index 1fa32599ab..a5a4da64cf 100644 --- a/app/executor.php +++ b/app/executor.php @@ -18,6 +18,7 @@ use Utopia\Orchestration\Orchestration; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\S3; use Utopia\Storage\Storage; use Utopia\Swoole\Request; @@ -129,6 +130,13 @@ function getStorageDevice($root): Device { $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); $doSpacesAcl = 'private'; return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_LINODE: + $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); + $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); + $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); + $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); + $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); } } diff --git a/app/init.php b/app/init.php index 9a73a7da10..0d9c1441c8 100644 --- a/app/init.php +++ b/app/init.php @@ -55,6 +55,7 @@ use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Linode; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -836,6 +837,13 @@ function getDevice($root): Device { $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); $doSpacesAcl = 'private'; return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_LINODE: + $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); + $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); + $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); + $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); + $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); } } diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 22b95fd2dd..6604737bef 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -110,6 +110,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -202,6 +206,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET - _APP_FUNCTIONS_CPUS - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP @@ -343,6 +351,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 963488a00a..605c307ced 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -10,6 +10,7 @@ use Executor\Executor; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Linode; use Utopia\Storage\Storage; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; @@ -563,6 +564,13 @@ class DeletesV1 extends Worker $doSpacesAcl = 'private'; $device = new DOSpaces(APP_STORAGE_UPLOADS . '/app-' . $projectId, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); break; + case Storage::DEVICE_LINODE: + $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); + $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); + $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); + $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); + $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); } $device->deletePath($document->getId()); diff --git a/composer.lock b/composer.lock index 42df642923..8ca96a1f56 100644 --- a/composer.lock +++ b/composer.lock @@ -3192,16 +3192,16 @@ }, { "name": "composer/semver", - "version": "3.3.0", + "version": "3.3.1", "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "f79c90ad4e9b41ac4dfc5d77bf398cf61fbd718b" + "reference": "5d8e574bb0e69188786b8ef77d43341222a41a71" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/f79c90ad4e9b41ac4dfc5d77bf398cf61fbd718b", - "reference": "f79c90ad4e9b41ac4dfc5d77bf398cf61fbd718b", + "url": "https://api.github.com/repos/composer/semver/zipball/5d8e574bb0e69188786b8ef77d43341222a41a71", + "reference": "5d8e574bb0e69188786b8ef77d43341222a41a71", "shasum": "" }, "require": { @@ -3253,7 +3253,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.3.0" + "source": "https://github.com/composer/semver/tree/3.3.1" }, "funding": [ { @@ -3269,7 +3269,7 @@ "type": "tidelift" } ], - "time": "2022-03-15T08:35:57+00:00" + "time": "2022-03-16T11:22:07+00:00" }, { "name": "composer/xdebug-handler", @@ -6580,5 +6580,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index ec77a2ece5..3c703e57cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,6 +123,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -279,6 +283,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -446,6 +454,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 9682688736..6c3263c188 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -12,6 +12,7 @@ use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\S3; use Exception; @@ -278,6 +279,13 @@ abstract class Worker $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); $doSpacesAcl = 'private'; return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_LINODE: + $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); + $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); + $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); + $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); + $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); } } } From 51ea3b2cbc1765cf5968b998132bbb15f24e62ff Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh <77877486+everly-gif@users.noreply.github.com> Date: Fri, 18 Mar 2022 22:17:09 +0530 Subject: [PATCH 02/17] Update deletes.php --- app/workers/deletes.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 605c307ced..b3641c58d4 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -571,6 +571,7 @@ class DeletesV1 extends Worker $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + break; } $device->deletePath($document->getId()); From fc8397de79fcf4c36ad7618ac5da5cf3659927e9 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh <77877486+everly-gif@users.noreply.github.com> Date: Fri, 18 Mar 2022 22:41:30 +0530 Subject: [PATCH 03/17] Update variables.php --- app/config/variables.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index 4d0f2ae117..36be9c9fc0 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -519,7 +519,7 @@ return [ ], [ 'name' => '_APP_STORAGE_LINODE_REGION', - 'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info for your space from Linode console.', + 'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info from your Linode console.', 'introduction' => '0.13.0', 'default' => 'us-eas-1', 'required' => false, From f03a68fbfece1f98df1302fe97e6a599667970a7 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Fri, 18 Mar 2022 17:17:43 +0000 Subject: [PATCH 04/17] add storage support for wasabi --- CHANGES.md | 2 +- Dockerfile | 4 ++++ app/config/variables.php | 32 ++++++++++++++++++++++++++++++++ app/executor.php | 8 ++++++++ app/init.php | 8 ++++++++ app/views/install/compose.phtml | 12 ++++++++++++ app/workers/deletes.php | 9 +++++++++ docker-compose.yml | 12 ++++++++++++ src/Appwrite/Resque/Worker.php | 8 ++++++++ 9 files changed, 94 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7b09391f49..4521093100 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ # Latest ## Features ### Storage -- Support for Linode object storage +- Support for Linode object storage and Wasabi # Version 0.13.3 ## Bugs diff --git a/Dockerfile b/Dockerfile index 204490f4d0..c776f42fac 100755 --- a/Dockerfile +++ b/Dockerfile @@ -166,6 +166,10 @@ ENV _APP_SERVER=swoole \ _APP_STORAGE_LINODE_SECRET= \ _APP_STORAGE_LINODE_REGION= \ _APP_STORAGE_LINODE_BUCKET= \ + _APP_STORAGE_WASABI_ACCESS_KEY= \ + _APP_STORAGE_WASABI_SECRET= \ + _APP_STORAGE_WASABI_REGION= \ + _APP_STORAGE_WASABI_BUCKET= \ _APP_REDIS_HOST=redis \ _APP_REDIS_PORT=6379 \ _APP_DB_HOST=mariadb \ diff --git a/app/config/variables.php b/app/config/variables.php index 4d0f2ae117..4270ef68c1 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -533,6 +533,38 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_STORAGE_WASABI_ACCESS_KEY', + 'description' => 'Wasabi access key. Required when the storage adapter is set to Wasabi. You can get your access key from your Wasabi console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_WASABI_SECRET', + 'description' => 'Wasabi secret key. Required when the storage adapter is set to Wasabi. You can get your secret key from your Wasabi console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_WASABI_REGION', + 'description' => 'Wasabi region. Required when storage adapter is set to Wasabi. You can find your region info from your Wasabi console.', + 'introduction' => '0.13.0', + 'default' => 'us-eas-1', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_WASABI_BUCKET', + 'description' => 'Wasabi bucket. Required when storage adapter is set to Wasabi. You can create buckets in your Wasabi console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], ], ], [ diff --git a/app/executor.php b/app/executor.php index a5a4da64cf..c23d57d758 100644 --- a/app/executor.php +++ b/app/executor.php @@ -19,6 +19,7 @@ use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; +use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Device\S3; use Utopia\Storage\Storage; use Utopia\Swoole\Request; @@ -137,6 +138,13 @@ function getStorageDevice($root): Device { $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + case Storage::DEVICE_WASABI: + $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); + $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); + $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); + $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); + $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } diff --git a/app/init.php b/app/init.php index 0d9c1441c8..7dbd12a502 100644 --- a/app/init.php +++ b/app/init.php @@ -56,6 +56,7 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; +use Utopia\Storage\Device\Wasabi; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -844,6 +845,13 @@ function getDevice($root): Device { $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + case Storage::DEVICE_WASABI: + $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); + $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); + $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); + $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); + $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 6604737bef..338df71f7c 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -114,6 +114,10 @@ services: - _APP_STORAGE_LINODE_SECRET - _APP_STORAGE_LINODE_REGION - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -210,6 +214,10 @@ services: - _APP_STORAGE_LINODE_SECRET - _APP_STORAGE_LINODE_REGION - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - _APP_FUNCTIONS_CPUS - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP @@ -355,6 +363,10 @@ services: - _APP_STORAGE_LINODE_SECRET - _APP_STORAGE_LINODE_REGION - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 605c307ced..8370d09e7e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -11,6 +11,7 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; +use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Storage; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; @@ -571,6 +572,14 @@ class DeletesV1 extends Worker $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + case Storage::DEVICE_WASABI: + $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); + $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); + $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); + $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); + $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + break; } $device->deletePath($document->getId()); diff --git a/docker-compose.yml b/docker-compose.yml index 3c703e57cb..647f727ac7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,6 +127,10 @@ services: - _APP_STORAGE_LINODE_SECRET - _APP_STORAGE_LINODE_REGION - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -287,6 +291,10 @@ services: - _APP_STORAGE_LINODE_SECRET - _APP_STORAGE_LINODE_REGION - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -458,6 +466,10 @@ services: - _APP_STORAGE_LINODE_SECRET - _APP_STORAGE_LINODE_REGION - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 6c3263c188..93f437fbf1 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -13,6 +13,7 @@ use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; +use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Device\S3; use Exception; @@ -286,6 +287,13 @@ abstract class Worker $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + case Storage::DEVICE_WASABI: + $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); + $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); + $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); + $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); + $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } } From 66d3f03f35a6a2dcb4244a9fc20f7283e8db8e9f Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Fri, 18 Mar 2022 17:25:09 +0000 Subject: [PATCH 05/17] update deletes.php --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b221ecec54..b5a69d6c8b 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -579,7 +579,7 @@ class DeletesV1 extends Worker $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + $device = new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); break; } From c407c836159558477e635a3c0a0535a3f8c25bdc Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh <77877486+everly-gif@users.noreply.github.com> Date: Fri, 18 Mar 2022 22:56:12 +0530 Subject: [PATCH 06/17] Update deletes.php --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b3641c58d4..eb80a5fc90 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -570,7 +570,7 @@ class DeletesV1 extends Worker $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + $device= new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); break; } From 7eb59c5290c788ff002ea9955313a12e7c39ebf5 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh <77877486+everly-gif@users.noreply.github.com> Date: Fri, 18 Mar 2022 22:58:28 +0530 Subject: [PATCH 07/17] Update deletes.php --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b5a69d6c8b..47f503ddf8 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -571,7 +571,7 @@ class DeletesV1 extends Worker $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + $device = new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); break; case Storage::DEVICE_WASABI: $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); From e9196bfce4c34a7cd16f88a1a69d6c3b7e89a434 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Wed, 4 May 2022 11:20:16 +0000 Subject: [PATCH 08/17] feat add backblaze support --- CHANGES.md | 1 + Dockerfile | 4 ++++ app/config/variables.php | 34 ++++++++++++++++++++++++++++++++- app/executor.php | 8 ++++++++ app/init.php | 8 ++++++++ app/views/install/compose.phtml | 12 ++++++++++++ app/workers/deletes.php | 9 +++++++++ docker-compose.yml | 12 ++++++++++++ src/Appwrite/Resque/Worker.php | 8 ++++++++ 9 files changed, 95 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7580ccb114..a90226173b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Unreleased Version - Renamed `providers` to `authProviders` in project collection **Breaking Change** +- Support for Backblaze adapter in Storage # Version 0.13.4 diff --git a/Dockerfile b/Dockerfile index c464885054..8df9166480 100755 --- a/Dockerfile +++ b/Dockerfile @@ -162,6 +162,10 @@ ENV _APP_SERVER=swoole \ _APP_STORAGE_DO_SPACES_SECRET= \ _APP_STORAGE_DO_SPACES_REGION= \ _APP_STORAGE_DO_SPACES_BUCKET= \ + _APP_STORAGE_BACKBLAZE_ACCESS_KEY= \ + _APP_STORAGE_BACKBLAZE_SECRET= \ + _APP_STORAGE_BACKBLAZE_REGION= \ + _APP_STORAGE_BACKBLAZE_BUCKET= \ _APP_REDIS_HOST=redis \ _APP_REDIS_PORT=6379 \ _APP_DB_HOST=mariadb \ diff --git a/app/config/variables.php b/app/config/variables.php index 2bed472758..0326b97903 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -440,7 +440,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DEVICE', - 'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\' and \'DOSpaces\'.', + 'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\', \'DOSpaces\' and \'Backblaze\'.', 'introduction' => '0.13.0', 'default' => 'Local', 'required' => false, @@ -510,6 +510,38 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_STORAGE_BACKBLAZE_ACCESS_KEY', + 'description' => 'Backblaze access key. Required when the storage adapter is set to BackBlaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_BACKBLAZE_SECRET', + 'description' => 'Backblaze secret key. Required when the storage adapter is set to BackBlaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_BACKBLAZE_REGION', + 'description' => 'Backblaze region. Required when storage adapter is set to BackBlaze. You can find your region info from your Backblaze console.', + 'introduction' => '0.13.0', + 'default' => 'us-west-004', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_STORAGE_BACKBLAZE_BUCKET', + 'description' => 'Backblaze bucket. Required when storage adapter is set to BackBlaze. You can create your bucket from your Backblaze console.', + 'introduction' => '0.13.0', + 'default' => '', + 'required' => false, + 'question' => '', + ], ], ], [ diff --git a/app/executor.php b/app/executor.php index e9f4c21391..1fd09c1ffe 100644 --- a/app/executor.php +++ b/app/executor.php @@ -17,6 +17,7 @@ use Utopia\Orchestration\Adapter\DockerCLI; use Utopia\Orchestration\Orchestration; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\BackBlaze; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\S3; use Utopia\Storage\Storage; @@ -130,6 +131,13 @@ function getStorageDevice($root): Device { $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); $doSpacesAcl = 'private'; return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + return new BackBlaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); } } diff --git a/app/init.php b/app/init.php index fd509297f6..259eae7bb8 100644 --- a/app/init.php +++ b/app/init.php @@ -53,6 +53,7 @@ use Utopia\Database\Query; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\BackBlaze; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; @@ -835,6 +836,13 @@ function getDevice($root): Device { $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); $doSpacesAcl = 'private'; return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + return new BackBlaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); } } diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 4c3406ffed..3c9c7f50a2 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -119,6 +119,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -266,6 +270,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -430,6 +438,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b8a44ebb82..fe383036c5 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -10,6 +10,7 @@ use Executor\Executor; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\BackBlaze; use Utopia\Storage\Storage; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; @@ -563,6 +564,14 @@ class DeletesV1 extends Worker $doSpacesAcl = 'private'; $device = new DOSpaces(APP_STORAGE_UPLOADS . '/app-' . $projectId, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); break; + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + $device= new BackBlaze(APP_STORAGE_UPLOADS . '/app-' . $projectId, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + break; } $device->deletePath($document->getId()); diff --git a/docker-compose.yml b/docker-compose.yml index 9f567b4233..8b1aa0c00d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,6 +141,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -304,6 +308,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -488,6 +496,10 @@ services: - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 9682688736..531d1a8007 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -12,6 +12,7 @@ use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\BackBlaze; use Utopia\Storage\Device\S3; use Exception; @@ -278,6 +279,13 @@ abstract class Worker $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); $doSpacesAcl = 'private'; return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + return new BackBlaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); } } } From 5e14ee478ebb2e2693be80d7c29c00f1f039341e Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Tue, 10 May 2022 11:15:56 +0000 Subject: [PATCH 09/17] change BackBlaze to Backblaze --- .env | 4 ++++ app/config/variables.php | 8 ++++---- app/controllers/shared/api.php | 1 + app/executor.php | 4 ++-- app/init.php | 4 ++-- app/workers/deletes.php | 4 ++-- src/Appwrite/Resque/Worker.php | 4 ++-- 7 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 1b35cca898..f250c3b1fa 100644 --- a/.env +++ b/.env @@ -32,6 +32,10 @@ _APP_STORAGE_DO_SPACES_ACCESS_KEY= _APP_STORAGE_DO_SPACES_SECRET= _APP_STORAGE_DO_SPACES_REGION=us-eas-1 _APP_STORAGE_DO_SPACES_BUCKET= +_APP_STORAGE_BACKBLAZE_ACCESS_KEY= +_APP_STORAGE_BACKBLAZE_SECRET= +_APP_STORAGE_BACKBLAZE_REGION=us-eas-1 +_APP_STORAGE_BACKBLAZE_BUCKET= _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 diff --git a/app/config/variables.php b/app/config/variables.php index 0326b97903..6b926a25e1 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -512,7 +512,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_ACCESS_KEY', - 'description' => 'Backblaze access key. Required when the storage adapter is set to BackBlaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.', + 'description' => 'Backblaze access key. Required when the storage adapter is set to Backblaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -520,7 +520,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_SECRET', - 'description' => 'Backblaze secret key. Required when the storage adapter is set to BackBlaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.', + 'description' => 'Backblaze secret key. Required when the storage adapter is set to Backblaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, @@ -528,7 +528,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_REGION', - 'description' => 'Backblaze region. Required when storage adapter is set to BackBlaze. You can find your region info from your Backblaze console.', + 'description' => 'Backblaze region. Required when storage adapter is set to Backblaze. You can find your region info from your Backblaze console.', 'introduction' => '0.13.0', 'default' => 'us-west-004', 'required' => false, @@ -536,7 +536,7 @@ return [ ], [ 'name' => '_APP_STORAGE_BACKBLAZE_BUCKET', - 'description' => 'Backblaze bucket. Required when storage adapter is set to BackBlaze. You can create your bucket from your Backblaze console.', + 'description' => 'Backblaze bucket. Required when storage adapter is set to Backblaze. You can create your bucket from your Backblaze console.', 'introduction' => '0.13.0', 'default' => '', 'required' => false, diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c5c6da6167..218a2e339c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -8,6 +8,7 @@ use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Database\Document; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Backblaze; use Utopia\Database\Validator\Authorization; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; diff --git a/app/executor.php b/app/executor.php index 1fd09c1ffe..3d827f0a24 100644 --- a/app/executor.php +++ b/app/executor.php @@ -17,7 +17,7 @@ use Utopia\Orchestration\Adapter\DockerCLI; use Utopia\Orchestration\Orchestration; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; -use Utopia\Storage\Device\BackBlaze; +use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\S3; use Utopia\Storage\Storage; @@ -137,7 +137,7 @@ function getStorageDevice($root): Device { $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); $backblazeAcl = 'private'; - return new BackBlaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); } } diff --git a/app/init.php b/app/init.php index 259eae7bb8..a87f3ee3fc 100644 --- a/app/init.php +++ b/app/init.php @@ -53,7 +53,7 @@ use Utopia\Database\Query; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; -use Utopia\Storage\Device\BackBlaze; +use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; @@ -842,7 +842,7 @@ function getDevice($root): Device { $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); $backblazeAcl = 'private'; - return new BackBlaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); } } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index fe383036c5..441aebf728 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -10,7 +10,7 @@ use Executor\Executor; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; -use Utopia\Storage\Device\BackBlaze; +use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Storage; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; @@ -570,7 +570,7 @@ class DeletesV1 extends Worker $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); $backblazeAcl = 'private'; - $device= new BackBlaze(APP_STORAGE_UPLOADS . '/app-' . $projectId, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + $device= new Backblaze(APP_STORAGE_UPLOADS . '/app-' . $projectId, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); break; } diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 531d1a8007..34a37dbc5e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -12,7 +12,7 @@ use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\DOSpaces; -use Utopia\Storage\Device\BackBlaze; +use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; use Exception; @@ -285,7 +285,7 @@ abstract class Worker $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); $backblazeAcl = 'private'; - return new BackBlaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); } } } From 9d43ad32b4996c54c2fb11bff29720deb15ad458 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Tue, 10 May 2022 11:19:58 +0000 Subject: [PATCH 10/17] change default env value --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index f250c3b1fa..1a4d64d8d1 100644 --- a/.env +++ b/.env @@ -34,7 +34,7 @@ _APP_STORAGE_DO_SPACES_REGION=us-eas-1 _APP_STORAGE_DO_SPACES_BUCKET= _APP_STORAGE_BACKBLAZE_ACCESS_KEY= _APP_STORAGE_BACKBLAZE_SECRET= -_APP_STORAGE_BACKBLAZE_REGION=us-eas-1 +_APP_STORAGE_BACKBLAZE_REGION=us-west-004 _APP_STORAGE_BACKBLAZE_BUCKET= _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav From 8cbb28fb989236c7aa4851cb96be2737e78f65ce Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Tue, 10 May 2022 15:22:58 +0000 Subject: [PATCH 11/17] use anchors and aliases to clean up compose file --- app/views/install/compose.phtml | 57 +++++++++++---------------------- docker-compose.yml | 57 +++++++++++---------------------- 2 files changed, 36 insertions(+), 78 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 3c9c7f50a2..fd8bb2086b 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -5,6 +5,21 @@ x-logging: &x-logging max-file: 5 max-size: 10m +x-env-storage: &x-env-storage |- + _APP_STORAGE_DEVICE + _APP_STORAGE_S3_ACCESS_KEY + _APP_STORAGE_S3_SECRET + _APP_STORAGE_S3_REGION + _APP_STORAGE_S3_BUCKET + _APP_STORAGE_DO_SPACES_ACCESS_KEY + _APP_STORAGE_DO_SPACES_SECRET + _APP_STORAGE_DO_SPACES_REGION + _APP_STORAGE_DO_SPACES_BUCKET + _APP_STORAGE_BACKBLAZE_ACCESS_KEY + _APP_STORAGE_BACKBLAZE_SECRET + _APP_STORAGE_BACKBLAZE_REGION + _APP_STORAGE_BACKBLAZE_BUCKET + getParam('httpPort', ''); @@ -110,19 +125,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET + - *x-env-storage - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -261,19 +264,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET + - *x-env-storage - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -429,19 +420,7 @@ services: - OPEN_RUNTIMES_NETWORK - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET + - *x-env-storage - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/docker-compose.yml b/docker-compose.yml index 8b1aa0c00d..377adb8402 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,21 @@ x-logging: &x-logging max-file: '5' max-size: '10m' +x-env-storage: &x-env-storage |- + _APP_STORAGE_DEVICE + _APP_STORAGE_S3_ACCESS_KEY + _APP_STORAGE_S3_SECRET + _APP_STORAGE_S3_REGION + _APP_STORAGE_S3_BUCKET + _APP_STORAGE_DO_SPACES_ACCESS_KEY + _APP_STORAGE_DO_SPACES_SECRET + _APP_STORAGE_DO_SPACES_REGION + _APP_STORAGE_DO_SPACES_BUCKET + _APP_STORAGE_BACKBLAZE_ACCESS_KEY + _APP_STORAGE_BACKBLAZE_SECRET + _APP_STORAGE_BACKBLAZE_REGION + _APP_STORAGE_BACKBLAZE_BUCKET + version: '3' services: @@ -132,19 +147,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET + - *x-env-storage - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -299,19 +302,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET + - *x-env-storage - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET @@ -487,19 +478,7 @@ services: - OPEN_RUNTIMES_NETWORK - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - _APP_STORAGE_DEVICE - - _APP_STORAGE_S3_ACCESS_KEY - - _APP_STORAGE_S3_SECRET - - _APP_STORAGE_S3_REGION - - _APP_STORAGE_S3_BUCKET - - _APP_STORAGE_DO_SPACES_ACCESS_KEY - - _APP_STORAGE_DO_SPACES_SECRET - - _APP_STORAGE_DO_SPACES_REGION - - _APP_STORAGE_DO_SPACES_BUCKET - - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - - _APP_STORAGE_BACKBLAZE_SECRET - - _APP_STORAGE_BACKBLAZE_REGION - - _APP_STORAGE_BACKBLAZE_BUCKET + - *x-env-storage - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD From 7350d4e4470b2b55158bf4f109a97a634ce86eac Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Thu, 12 May 2022 14:22:41 +0000 Subject: [PATCH 12/17] better way to initialize storage libs --- app/executor.php | 37 ++++++--------------------------- app/init.php | 38 +++++++--------------------------- app/workers/deletes.php | 30 ++------------------------- src/Appwrite/Resque/Worker.php | 2 +- 4 files changed, 16 insertions(+), 91 deletions(-) diff --git a/app/executor.php b/app/executor.php index 3d827f0a24..6fc1aaf691 100644 --- a/app/executor.php +++ b/app/executor.php @@ -2,6 +2,7 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Runtimes\Runtimes; +use Appwrite\Resque\Worker; use Swoole\ConnectionPool; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; @@ -113,34 +114,6 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null) Console::error('[Error] Line: ' . $error->getLine()); }; -function getStorageDevice($root): Device { - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { - case Storage::DEVICE_LOCAL:default: - return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - } -} - App::post('/v1/runtimes') ->desc("Create a new runtime server") ->param('runtimeId', '', new Text(64), 'Unique runtime ID.') @@ -181,7 +154,8 @@ App::post('/v1/runtimes') /** * Copy code files from source to a temporary location on the executor */ - $sourceDevice = getStorageDevice("/"); + $worker=new Worker(); + $sourceDevice = $worker->getDevice("/"); $localDevice = new Local(); $buffer = $sourceDevice->read($source); if(!$localDevice->write($tmpSource, $buffer)) { @@ -268,8 +242,9 @@ App::post('/v1/runtimes') if (!\file_exists($tmpBuild)) { throw new Exception('Something went wrong during the build process', 500); } - - $destinationDevice = getStorageDevice($destination); + + $worker=new Worker(); + $destinationDevice = $worker->getDevice($destination); $outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $buffer = $localDevice->read($tmpBuild); diff --git a/app/init.php b/app/init.php index a87f3ee3fc..e3b364b0e6 100644 --- a/app/init.php +++ b/app/init.php @@ -27,6 +27,7 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\Resque\Worker; use Appwrite\Stats\Stats; use Appwrite\Utopia\View; use Utopia\App; @@ -807,45 +808,20 @@ App::setResource('deviceLocal', function() { }); App::setResource('deviceFiles', function($project) { - return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); + $worker=new Worker(); + return $worker->getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); App::setResource('deviceFunctions', function($project) { - return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); + $worker=new Worker(); + return $worker->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); App::setResource('deviceBuilds', function($project) { - return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); + $worker=new Worker(); + return $worker->getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); }, ['project']); -function getDevice($root): Device { - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { - case Storage::DEVICE_LOCAL:default: - return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - } -} - App::setResource('mode', function($request) { /** @var Appwrite\Utopia\Request $request */ diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 441aebf728..b365adea81 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -545,34 +545,8 @@ class DeletesV1 extends Worker $dbForProject = $this->getProjectDB($projectId); $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); - $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); - - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - $device = new S3(APP_STORAGE_UPLOADS . '/app-' . $projectId, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - break; - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - $device = new DOSpaces(APP_STORAGE_UPLOADS . '/app-' . $projectId, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - break; - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - $device= new Backblaze(APP_STORAGE_UPLOADS . '/app-' . $projectId, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - break; - } + $worker=new Worker(); + $device=$worker->getDevice(APP_STORAGE_UPLOADS.'/app-'.$projectId); $device->deletePath($document->getId()); } diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 34a37dbc5e..40fad72102 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -260,7 +260,7 @@ abstract class Worker * @param string $root path of the device * @return Device */ - private function getDevice($root): Device + public function getDevice($root): Device { switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { case Storage::DEVICE_LOCAL:default: From a45fab4e43e8eef91230d50af1e4da0ae2692382 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Thu, 12 May 2022 22:01:53 +0000 Subject: [PATCH 13/17] better way to initialize storage libs --- app/executor.php | 37 ++++++++++++++++++++++++++++------ app/init.php | 44 +++++++++++++++++++++++++++++++---------- app/workers/deletes.php | 3 +-- 3 files changed, 66 insertions(+), 18 deletions(-) diff --git a/app/executor.php b/app/executor.php index 6fc1aaf691..3d827f0a24 100644 --- a/app/executor.php +++ b/app/executor.php @@ -2,7 +2,6 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Runtimes\Runtimes; -use Appwrite\Resque\Worker; use Swoole\ConnectionPool; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; @@ -114,6 +113,34 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null) Console::error('[Error] Line: ' . $error->getLine()); }; +function getStorageDevice($root): Device { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local($root); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + } +} + App::post('/v1/runtimes') ->desc("Create a new runtime server") ->param('runtimeId', '', new Text(64), 'Unique runtime ID.') @@ -154,8 +181,7 @@ App::post('/v1/runtimes') /** * Copy code files from source to a temporary location on the executor */ - $worker=new Worker(); - $sourceDevice = $worker->getDevice("/"); + $sourceDevice = getStorageDevice("/"); $localDevice = new Local(); $buffer = $sourceDevice->read($source); if(!$localDevice->write($tmpSource, $buffer)) { @@ -242,9 +268,8 @@ App::post('/v1/runtimes') if (!\file_exists($tmpBuild)) { throw new Exception('Something went wrong during the build process', 500); } - - $worker=new Worker(); - $destinationDevice = $worker->getDevice($destination); + + $destinationDevice = getStorageDevice($destination); $outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $buffer = $localDevice->read($tmpBuild); diff --git a/app/init.php b/app/init.php index e3b364b0e6..170542effa 100644 --- a/app/init.php +++ b/app/init.php @@ -27,7 +27,6 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; -use Appwrite\Resque\Worker; use Appwrite\Stats\Stats; use Appwrite\Utopia\View; use Utopia\App; @@ -53,10 +52,10 @@ use Swoole\Database\RedisPool; use Utopia\Database\Query; use Utopia\Storage\Device; use Utopia\Storage\Storage; -use Utopia\Storage\Device\Local; use Utopia\Storage\Device\Backblaze; -use Utopia\Storage\Device\S3; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\S3; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -808,20 +807,45 @@ App::setResource('deviceLocal', function() { }); App::setResource('deviceFiles', function($project) { - $worker=new Worker(); - return $worker->getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); + return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); App::setResource('deviceFunctions', function($project) { - $worker=new Worker(); - return $worker->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); + return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); App::setResource('deviceBuilds', function($project) { - $worker=new Worker(); - return $worker->getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); + return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); }, ['project']); +function getDevice($root): Device { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local($root); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + } +} + App::setResource('mode', function($request) { /** @var Appwrite\Utopia\Request $request */ @@ -836,4 +860,4 @@ App::setResource('mode', function($request) { App::setResource('geodb', function($register) { /** @var Utopia\Registry\Registry $register */ return $register->get('geodb'); -}, ['register']); +}, ['register']); \ No newline at end of file diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b365adea81..f6d2a94960 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -545,8 +545,7 @@ class DeletesV1 extends Worker $dbForProject = $this->getProjectDB($projectId); $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); - $worker=new Worker(); - $device=$worker->getDevice(APP_STORAGE_UPLOADS.'/app-'.$projectId); + $device=$this->getDevice(APP_STORAGE_UPLOADS.'/app-'.$projectId); $device->deletePath($document->getId()); } From f9da67ec3ea2d23c6b52f10b6dd14db5f58b328b Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Thu, 12 May 2022 22:33:08 +0000 Subject: [PATCH 14/17] format code --- app/init.php | 2 +- app/workers/deletes.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 170542effa..adbcab4aec 100644 --- a/app/init.php +++ b/app/init.php @@ -860,4 +860,4 @@ App::setResource('mode', function($request) { App::setResource('geodb', function($register) { /** @var Utopia\Registry\Registry $register */ return $register->get('geodb'); -}, ['register']); \ No newline at end of file +}, ['register']); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f6d2a94960..caac401f4e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -545,7 +545,7 @@ class DeletesV1 extends Worker $dbForProject = $this->getProjectDB($projectId); $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); - $device=$this->getDevice(APP_STORAGE_UPLOADS.'/app-'.$projectId); + $device = $this->getDevice(APP_STORAGE_UPLOADS.'/app-'.$projectId); $device->deletePath($document->getId()); } From 36873be7ea6f4cf3d03b6d5c4d1fbedf4c2578d8 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Mon, 16 May 2022 09:31:09 +0000 Subject: [PATCH 15/17] add default region value for linode --- app/config/variables.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index 91be780fa2..961b4abd6f 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -562,7 +562,7 @@ return [ 'name' => '_APP_STORAGE_LINODE_REGION', 'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info from your Linode console.', 'introduction' => '0.13.0', - 'default' => 'us-eas-1', + 'default' => 'eu-central-1', 'required' => false, 'question' => '', ], From c329b487a9ed4c5c4d6616fc817ff085e90e2c3e Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Mon, 16 May 2022 10:05:31 +0000 Subject: [PATCH 16/17] remove uneccessary use statement --- app/config/variables.php | 2 +- app/controllers/shared/api.php | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 396a120cd5..6df009c4fc 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -440,7 +440,7 @@ return [ ], [ 'name' => '_APP_STORAGE_DEVICE', - 'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\', \'DOSpaces\', \'Backblaze\' and \'Linode\'.', + 'description' => 'Select default storage device. The default value is \'Local\'. List of supported adapters are \'Local\', \'S3\', \'DOSpaces\', \'Backblaze\', \'Linode\' and \'Wasabi\'.', 'introduction' => '0.13.0', 'default' => 'Local', 'required' => false, diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 218a2e339c..c5c6da6167 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -8,7 +8,6 @@ use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Database\Document; use Utopia\Storage\Device\DOSpaces; -use Utopia\Storage\Device\Backblaze; use Utopia\Database\Validator\Authorization; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; From 90a4f4d756eba2e05313030aeac257bdf208f7be Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 19 May 2022 13:07:40 +0200 Subject: [PATCH 17/17] chore: update composer --- composer.json | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 1cd1236d0f..79f144c068 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.3.*", - "utopia-php/storage": "0.7.*", + "utopia-php/storage": "0.9.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.4.*", diff --git a/composer.lock b/composer.lock index 69aadee3ae..ece1c7694e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "69e0c12a4c4b22cf17727f5b8833b1ac", + "content-hash": "78be7780db881871009109a373b2fc35", "packages": [ { "name": "adhocore/jwt", @@ -2628,16 +2628,16 @@ }, { "name": "utopia-php/storage", - "version": "0.7.1", + "version": "0.9.0", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "1921d5da3d155c1e03b26f8f6184dba3a69cd5e4" + "reference": "c7912481a56e17cc86358fa8de57309de5e88ef7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/1921d5da3d155c1e03b26f8f6184dba3a69cd5e4", - "reference": "1921d5da3d155c1e03b26f8f6184dba3a69cd5e4", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/c7912481a56e17cc86358fa8de57309de5e88ef7", + "reference": "c7912481a56e17cc86358fa8de57309de5e88ef7", "shasum": "" }, "require": { @@ -2674,9 +2674,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.7.1" + "source": "https://github.com/utopia-php/storage/tree/0.9.0" }, - "time": "2022-02-20T13:27:43+00:00" + "time": "2022-05-19T11:05:45+00:00" }, { "name": "utopia-php/swoole",