From 503495df16485aab68b27fdb964861db19b830c3 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 15 May 2024 18:32:37 +0530 Subject: [PATCH 1/7] Update executor image name to exc-1 --- app/views/install/compose.phtml | 2 +- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 53e7e71518..f7fa18e3c0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -727,7 +727,7 @@ services: openruntimes-executor: container_name: openruntimes-executor - hostname: appwrite-executor + hostname: exc-1 <<: *x-logging restart: unless-stopped stop_signal: SIGINT diff --git a/docker-compose.yml b/docker-compose.yml index 82c38ba269..f933d0a8c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -791,7 +791,7 @@ services: openruntimes-executor: container_name: openruntimes-executor - hostname: appwrite-executor + hostname: exc-1 <<: *x-logging stop_signal: SIGINT image: openruntimes/executor:0.5.1 @@ -857,7 +857,7 @@ services: - OPR_PROXY_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG - OPR_PROXY_ALGORITHM=random - - OPR_PROXY_EXECUTORS=appwrite-executor + - OPR_PROXY_EXECUTORS=exc-1 - OPR_PROXY_HEALTHCHECK_INTERVAL=10000 - OPR_PROXY_MAX_TIMEOUT=600 - OPR_PROXY_HEALTHCHECK=enabled From d54986dc756fb68ea0dd32a2ed125f083eaaf77d Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Thu, 16 May 2024 05:31:44 +0000 Subject: [PATCH 2/7] fix(messaging): fix TextMagic class name The class name in utopia-php/mssaging is TextMagic so use the same here to prevent an error like: > Class "Utopia\Messaging\Adapter\SMS\Textmagic" not found ... --- src/Appwrite/Platform/Workers/Messaging.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index c270018430..0399302e77 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -25,7 +25,7 @@ use Utopia\Messaging\Adapter\SMS as SMSAdapter; use Utopia\Messaging\Adapter\SMS\Mock; use Utopia\Messaging\Adapter\SMS\Msg91; use Utopia\Messaging\Adapter\SMS\Telesign; -use Utopia\Messaging\Adapter\SMS\Textmagic; +use Utopia\Messaging\Adapter\SMS\TextMagic; use Utopia\Messaging\Adapter\SMS\Twilio; use Utopia\Messaging\Adapter\SMS\Vonage; use Utopia\Messaging\Messages\Email; @@ -459,7 +459,7 @@ class Messaging extends Action return match ($provider->getAttribute('provider')) { 'mock' => new Mock('username', 'password'), 'twilio' => new Twilio($credentials['accountSid'], $credentials['authToken']), - 'textmagic' => new Textmagic($credentials['username'], $credentials['apiKey']), + 'textmagic' => new TextMagic($credentials['username'], $credentials['apiKey']), 'telesign' => new Telesign($credentials['customerId'], $credentials['apiKey']), 'msg91' => new Msg91($credentials['senderId'], $credentials['authKey'], $credentials['templateId']), 'vonage' => new Vonage($credentials['apiKey'], $credentials['apiSecret']), From 8ecc2ee787962a9f24d4548b11406770f020dcfb Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 16 May 2024 14:03:51 +0530 Subject: [PATCH 3/7] Remove hyphen --- app/views/install/compose.phtml | 2 +- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index f7fa18e3c0..711b46733f 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -727,7 +727,7 @@ services: openruntimes-executor: container_name: openruntimes-executor - hostname: exc-1 + hostname: exc1 <<: *x-logging restart: unless-stopped stop_signal: SIGINT diff --git a/docker-compose.yml b/docker-compose.yml index f933d0a8c2..b3b197381b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -791,7 +791,7 @@ services: openruntimes-executor: container_name: openruntimes-executor - hostname: exc-1 + hostname: exc1 <<: *x-logging stop_signal: SIGINT image: openruntimes/executor:0.5.1 @@ -857,7 +857,7 @@ services: - OPR_PROXY_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG - OPR_PROXY_ALGORITHM=random - - OPR_PROXY_EXECUTORS=exc-1 + - OPR_PROXY_EXECUTORS=exc1 - OPR_PROXY_HEALTHCHECK_INTERVAL=10000 - OPR_PROXY_MAX_TIMEOUT=600 - OPR_PROXY_HEALTHCHECK=enabled From 60179b1790ba7fcfa70783d75b2e430824d4caa5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 17 May 2024 14:54:40 +1200 Subject: [PATCH 4/7] Fix not account for abuse/audit on project delete --- src/Appwrite/Platform/Workers/Deletes.php | 37 +++++++++++++---------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b08bfb027a..36d9dbd4f0 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -443,7 +443,7 @@ class Deletes extends Action * @param Document $document * @return void * @throws Authorization - * @throws \Utopia\Database\Exception + * @throws DatabaseException * @throws Conflict * @throws Restricted * @throws Structure @@ -471,11 +471,10 @@ class Deletes extends Action * @return void * @throws Exception * @throws Authorization - * @throws \Utopia\Database\Exception + * @throws DatabaseException */ private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void { - $projectId = $document->getId(); $projectInternalId = $document->getInternalId(); try { @@ -486,24 +485,18 @@ class Deletes extends Action } $dbForProject = $getProjectDB($document); - $projectCollectionIds = \array_keys(Config::getParam('collections', [])['projects']); + + $projectCollectionIds = [ + ...\array_keys(Config::getParam('collections', [])['projects']), + Audit::COLLECTION, + TimeLimit::COLLECTION, + ]; + $limit = \count($projectCollectionIds) + 25; while (true) { $collections = $dbForProject->listCollections($limit); - if ($dsn->getHost() === DATABASE_SHARED_TABLES) { - $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections); - - if ($collectionsIds == $projectCollectionIds) { - break; - } - } else { - if (empty($collections)) { - break; - } - } - foreach ($collections as $collection) { if ($dsn->getHost() !== DATABASE_SHARED_TABLES || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); @@ -511,6 +504,16 @@ class Deletes extends Action $this->deleteByGroup($collection->getId(), [], database: $dbForProject); } } + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections); + + if (empty(\array_diff($collectionsIds, $projectCollectionIds))) { + break; + } + } elseif (empty($collections)) { + break; + } } // Delete Platforms @@ -553,6 +556,8 @@ class Deletes extends Action // Delete metadata table if ($dsn->getHost() !== DATABASE_SHARED_TABLES) { $dbForProject->deleteCollection('_metadata'); + } else { + $this->deleteByGroup('_metadata', [], $dbForProject); } // Delete all storage directories From 6a9e98365177e48bdbcb800681a90700458f06d5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 17 May 2024 16:38:40 +1200 Subject: [PATCH 5/7] Catch DB errors on delete --- src/Appwrite/Platform/Workers/Deletes.php | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b08bfb027a..3320d05efd 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -679,9 +679,11 @@ class Deletes extends Action $dbForProject = $getProjectDB($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); - $status = $abuse->cleanup($abuseRetention); - if (!$status) { - throw new Exception('Failed to delete Abuse logs for project ' . $projectId); + + try { + $abuse->cleanup($abuseRetention); + } catch (DatabaseException $e) { + Console::error('Failed to delete abuse logs for project ' . $projectId . ': ' . $e->getMessage()); } } @@ -697,9 +699,11 @@ class Deletes extends Action $projectId = $project->getId(); $dbForProject = $getProjectDB($project); $audit = new Audit($dbForProject); - $status = $audit->cleanup($auditRetention); - if (!$status) { - throw new Exception('Failed to delete Audit logs for project' . $projectId); + + try { + $audit->cleanup($auditRetention); + } catch (DatabaseException $e) { + Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); } } @@ -954,7 +958,12 @@ class Deletes extends Action while ($sum === $limit) { $chunk++; - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + try { + $results = $database->find($collection, [Query::limit($limit), ...$queries]); + } catch (DatabaseException $e) { + Console::error('Failed to find documents for collection ' . $collection . ': ' . $e->getMessage()); + return; + } $sum = count($results); From deacaa2d6fd920e80f4d1bbb2c671346037b8bcc Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 17 May 2024 18:01:33 +0900 Subject: [PATCH 6/7] Multiple Changes, See Desc - Update Logger from 0.3.* to 0.5.* - Add old sentry syntax conversion - Add sampler for experiment logger, and set it to 4% --- app/controllers/general.php | 1 + app/init.php | 10 ++++++ composer.json | 2 +- composer.lock | 64 ++++++++++++++++++------------------- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 3734b81a3b..1fa70ed703 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -725,6 +725,7 @@ App::error() $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName); $adapter = new $classname($providerConfig); $logger = new Logger($adapter); + $logger->setSample(0.04); $publish = true; } } diff --git a/app/init.php b/app/init.php index fbb3e2ee8d..88e8de0ea2 100644 --- a/app/init.php +++ b/app/init.php @@ -735,6 +735,16 @@ $register->set('logger', function () { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled"); } + // Old Sentry Format conversion. Fallback until the old syntax is completely deprecated. + if (str_contains($providerConfig, ';') && strtolower($providerName) == 'sentry') { + $configChunks = \explode(";", $providerConfig); + + $sentryKey = $configChunks[0]; + $projectId = $configChunks[1]; + + $providerConfig = 'https://' . $sentryKey . '@sentry.io/' . $projectId; + } + $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName); $adapter = new $classname($providerConfig); return new Logger($adapter); diff --git a/composer.json b/composer.json index 8865a32d91..e247799645 100644 --- a/composer.json +++ b/composer.json @@ -57,7 +57,7 @@ "utopia-php/fetch": "0.2.*", "utopia-php/image": "0.6.*", "utopia-php/locale": "0.4.*", - "utopia-php/logger": "0.3.*", + "utopia-php/logger": "0.5.*", "utopia-php/messaging": "0.10.*", "utopia-php/migration": "0.4.*", "utopia-php/orchestration": "0.9.*", diff --git a/composer.lock b/composer.lock index 83c5e027b9..4044508d4f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6ce62f5b54254e5023c5ace349a0ced7", + "content-hash": "e3975335737921ddfe4e35d891ea222a", "packages": [ { "name": "adhocore/jwt", @@ -822,16 +822,16 @@ }, { "name": "paragonie/constant_time_encoding", - "version": "v2.6.3", + "version": "v2.7.0", "source": { "type": "git", "url": "https://github.com/paragonie/constant_time_encoding.git", - "reference": "58c3f47f650c94ec05a151692652a868995d2938" + "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/58c3f47f650c94ec05a151692652a868995d2938", - "reference": "58c3f47f650c94ec05a151692652a868995d2938", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/52a0d99e69f56b9ec27ace92ba56897fe6993105", + "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105", "shasum": "" }, "require": { @@ -885,7 +885,7 @@ "issues": "https://github.com/paragonie/constant_time_encoding/issues", "source": "https://github.com/paragonie/constant_time_encoding" }, - "time": "2022-06-14T06:56:20+00:00" + "time": "2024-05-08T12:18:48+00:00" }, { "name": "phpmailer/phpmailer", @@ -1902,22 +1902,23 @@ }, { "name": "utopia-php/logger", - "version": "0.3.2", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/logger.git", - "reference": "ba763c10688fe2ed715ad2bed3f13d18dfec6253" + "reference": "048042f9440a4517332872a9fa35e145e4aa5daf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/logger/zipball/ba763c10688fe2ed715ad2bed3f13d18dfec6253", - "reference": "ba763c10688fe2ed715ad2bed3f13d18dfec6253", + "url": "https://api.github.com/repos/utopia-php/logger/zipball/048042f9440a4517332872a9fa35e145e4aa5daf", + "reference": "048042f9440a4517332872a9fa35e145e4aa5daf", "shasum": "" }, "require": { "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" @@ -1949,9 +1950,9 @@ ], "support": { "issues": "https://github.com/utopia-php/logger/issues", - "source": "https://github.com/utopia-php/logger/tree/0.3.2" + "source": "https://github.com/utopia-php/logger/tree/0.5.1" }, - "time": "2023-11-22T14:45:43+00:00" + "time": "2024-05-17T07:57:39+00:00" }, { "name": "utopia-php/messaging", @@ -2005,22 +2006,21 @@ }, { "name": "utopia-php/migration", - "version": "0.4.1", + "version": "0.4.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "ae3cfe93f6d313105d226aeb68806660c806a925" + "reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/ae3cfe93f6d313105d226aeb68806660c806a925", - "reference": "ae3cfe93f6d313105d226aeb68806660c806a925", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/a8a5d392bebf082faf289f4dfe09d9fd76844c33", + "reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33", "shasum": "" }, "require": { "appwrite/appwrite": "10.1.0", - "php": "8.*", - "utopia-php/cli": "0.*" + "php": "8.*" }, "require-dev": { "laravel/pint": "1.*", @@ -2047,9 +2047,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.4.1" + "source": "https://github.com/utopia-php/migration/tree/0.4.4" }, - "time": "2024-05-01T13:19:18+00:00" + "time": "2024-05-17T05:25:31+00:00" }, { "name": "utopia-php/mongo", @@ -2823,16 +2823,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.38.2", + "version": "0.38.5", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "51284668529e2b10ed933412a42b603c76cded23" + "reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/51284668529e2b10ed933412a42b603c76cded23", - "reference": "51284668529e2b10ed933412a42b603c76cded23", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/830a46cc8e34ee096a76d4af6f00adf008a7cbf8", + "reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8", "shasum": "" }, "require": { @@ -2868,9 +2868,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.38.2" + "source": "https://github.com/appwrite/sdk-generator/tree/0.38.5" }, - "time": "2024-04-25T07:49:29+00:00" + "time": "2024-05-17T00:59:59+00:00" }, { "name": "doctrine/deprecations", @@ -3660,16 +3660,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.28.0", + "version": "1.29.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb" + "reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb", - "reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/536889f2b340489d328f5ffb7b02bb6b183ddedc", + "reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc", "shasum": "" }, "require": { @@ -3701,9 +3701,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.28.0" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.29.0" }, - "time": "2024-04-03T18:51:33+00:00" + "time": "2024-05-06T12:04:23+00:00" }, { "name": "phpunit/php-code-coverage", From 51a27315330c1a8f621d46546c1f3171af09940b Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 17 May 2024 18:36:25 +0900 Subject: [PATCH 7/7] Update composer.lock --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 4044508d4f..7fc372d624 100644 --- a/composer.lock +++ b/composer.lock @@ -1902,16 +1902,16 @@ }, { "name": "utopia-php/logger", - "version": "0.5.1", + "version": "0.5.2", "source": { "type": "git", "url": "https://github.com/utopia-php/logger.git", - "reference": "048042f9440a4517332872a9fa35e145e4aa5daf" + "reference": "c6dfdb672e41364c309b0c30dc03bc6d45446dba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/logger/zipball/048042f9440a4517332872a9fa35e145e4aa5daf", - "reference": "048042f9440a4517332872a9fa35e145e4aa5daf", + "url": "https://api.github.com/repos/utopia-php/logger/zipball/c6dfdb672e41364c309b0c30dc03bc6d45446dba", + "reference": "c6dfdb672e41364c309b0c30dc03bc6d45446dba", "shasum": "" }, "require": { @@ -1950,9 +1950,9 @@ ], "support": { "issues": "https://github.com/utopia-php/logger/issues", - "source": "https://github.com/utopia-php/logger/tree/0.5.1" + "source": "https://github.com/utopia-php/logger/tree/0.5.2" }, - "time": "2024-05-17T07:57:39+00:00" + "time": "2024-05-17T09:32:59+00:00" }, { "name": "utopia-php/messaging",