From 4fafb47508ae085e523b30f0c9ce330053e47e56 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Mon, 26 May 2025 17:42:54 +0200 Subject: [PATCH 01/17] Bring back telemetry for storage. This reverts commit 6906a97c224b9b7b923b1b8862ec0acd712d7125. --- app/init/resources.php | 45 ++++++++++++++++++++++-------------------- app/worker.php | 38 ++++++++++++++++++----------------- composer.lock | 12 +++++------ 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index c75df2a362..6ca0522b6c 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -52,6 +52,7 @@ use Utopia\Storage\Device\S3; use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Storage; use Utopia\System\System; +use Utopia\Telemetry\Adapter as Telemetry; use Utopia\Telemetry\Adapter\None as NoTelemetry; use Utopia\Validator\Hostname; use Utopia\Validator\WhiteList; @@ -454,7 +455,7 @@ App::setResource('getLogsDB', function (Group $pools, Cache $cache) { App::setResource('telemetry', fn () => new NoTelemetry()); -App::setResource('cache', function (Group $pools) { +App::setResource('cache', function (Group $pools, Telemetry $telemetry) { $list = Config::getParam('pools-cache', []); $adapters = []; @@ -462,8 +463,10 @@ App::setResource('cache', function (Group $pools) { $adapters[] = new CachePool($pools->get($value)); } - return new Cache(new Sharding($adapters)); -}, ['pools']); + $cache = new Cache(new Sharding($adapters)); + $cache->setTelemetry($telemetry); + return $cache; +}, ['pools', 'telemetry']); App::setResource('redis', function () { $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); @@ -486,24 +489,24 @@ App::setResource('timelimit', function (\Redis $redis) { }; }, ['redis']); -App::setResource('deviceForLocal', function () { - return new Local(); -}); -App::setResource('deviceForFiles', function ($project) { - return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); -}, ['project']); -App::setResource('deviceForSites', function ($project) { - return getDevice(APP_STORAGE_SITES . '/app-' . $project->getId()); -}, ['project']); -App::setResource('deviceForImports', function ($project) { - return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); -}, ['project']); -App::setResource('deviceForFunctions', function ($project) { - return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); -}, ['project']); -App::setResource('deviceForBuilds', function ($project) { - return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); -}, ['project']); +App::setResource('deviceForLocal', function (Telemetry $telemetry) { + return new Device\Telemetry($telemetry, new Local()); +}, ['telemetry']); +App::setResource('deviceForFiles', function ($project, Telemetry $telemetry) { + return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); +App::setResource('deviceForSites', function ($project, Telemetry $telemetry) { + return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); +}, ['project', 'telemetry']); +App::setResource('deviceForImports', function ($project, Telemetry $telemetry) { + return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); +App::setResource('deviceForFunctions', function ($project, Telemetry $telemetry) { + return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); +App::setResource('deviceForBuilds', function ($project, Telemetry $telemetry) { + return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); function getDevice(string $root, string $connection = ''): Device { diff --git a/app/worker.php b/app/worker.php index 597e8a9943..c92c48936f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -40,7 +40,9 @@ use Utopia\Queue\Message; use Utopia\Queue\Publisher; use Utopia\Queue\Server; use Utopia\Registry\Registry; +use Utopia\Storage\Device\Telemetry as TelemetryDevice; use Utopia\System\System; +use Utopia\Telemetry\Adapter as Telemetry; use Utopia\Telemetry\Adapter\None as NoTelemetry; Authorization::disable(); @@ -311,29 +313,29 @@ Server::setResource('pools', function (Registry $register) { Server::setResource('telemetry', fn () => new NoTelemetry()); -Server::setResource('deviceForSites', function (Document $project) { - return getDevice(APP_STORAGE_SITES . '/app-' . $project->getId()); -}, ['project']); +Server::setResource('deviceForSites', function (Document $project, Telemetry $telemetry) { + return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); +}, ['project', 'telemetry']); -Server::setResource('deviceForImports', function (Document $project) { - return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); -}, ['project']); +Server::setResource('deviceForImports', function (Document $project, Telemetry $telemetry) { + return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); -Server::setResource('deviceForFunctions', function (Document $project) { - return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); -}, ['project']); +Server::setResource('deviceForFunctions', function (Document $project, Telemetry $telemetry) { + return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); -Server::setResource('deviceForFiles', function (Document $project) { - return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); -}, ['project']); +Server::setResource('deviceForFiles', function (Document $project, Telemetry $telemetry) { + return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); -Server::setResource('deviceForBuilds', function (Document $project) { - return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); -}, ['project']); +Server::setResource('deviceForBuilds', function (Document $project, Telemetry $telemetry) { + return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())); +}, ['project', 'telemetry']); -Server::setResource('deviceForCache', function (Document $project) { - return getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); -}, ['project']); +Server::setResource('deviceForCache', function (Document $project, Telemetry $telemetry) { + return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId())); +}, ['project', 'telemetry']); Server::setResource( 'isResourceBlocked', diff --git a/composer.lock b/composer.lock index 875f012fc9..0ffe052061 100644 --- a/composer.lock +++ b/composer.lock @@ -4378,16 +4378,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.12", + "version": "0.18.13", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "9a2556c39b5f4d9f8e79111fd34ec889b7bb1e97" + "reference": "3d8ce53ae042173bf230445e996056c5f65ded22" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/9a2556c39b5f4d9f8e79111fd34ec889b7bb1e97", - "reference": "9a2556c39b5f4d9f8e79111fd34ec889b7bb1e97", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/3d8ce53ae042173bf230445e996056c5f65ded22", + "reference": "3d8ce53ae042173bf230445e996056c5f65ded22", "shasum": "" }, "require": { @@ -4430,9 +4430,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.12" + "source": "https://github.com/utopia-php/storage/tree/0.18.13" }, - "time": "2025-05-15T07:55:58+00:00" + "time": "2025-05-26T13:10:35+00:00" }, { "name": "utopia-php/swoole", From 8d5d9dc463c4995fba597bb7f7f0d0b3edc268d1 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Mon, 14 Apr 2025 12:17:12 +0200 Subject: [PATCH 02/17] chore: use exec to replace /bin/sh shell with the actual program --- bin/doctor | 2 +- bin/install | 2 +- bin/maintenance | 2 +- bin/migrate | 2 +- bin/queue-count-failed | 2 +- bin/queue-count-processing | 2 +- bin/queue-count-success | 2 +- bin/queue-retry | 2 +- bin/realtime | 2 +- bin/schedule-executions | 2 +- bin/schedule-functions | 2 +- bin/schedule-messages | 2 +- bin/sdks | 2 +- bin/specs | 2 +- bin/ssl | 2 +- bin/stats-resources | 2 +- bin/test | 2 +- bin/upgrade | 2 +- bin/vars | 2 +- bin/worker-audits | 2 +- bin/worker-builds | 2 +- bin/worker-certificates | 2 +- bin/worker-databases | 2 +- bin/worker-deletes | 2 +- bin/worker-functions | 2 +- bin/worker-mails | 2 +- bin/worker-messaging | 2 +- bin/worker-migrations | 2 +- bin/worker-stats-resources | 2 +- bin/worker-stats-usage | 2 +- bin/worker-webhooks | 2 +- 31 files changed, 31 insertions(+), 31 deletions(-) diff --git a/bin/doctor b/bin/doctor index b2a4547156..92b3cf6bf8 100755 --- a/bin/doctor +++ b/bin/doctor @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php doctor $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php doctor $@ diff --git a/bin/install b/bin/install index e669e91e6b..126d5373bf 100755 --- a/bin/install +++ b/bin/install @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php install $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php install $@ diff --git a/bin/maintenance b/bin/maintenance index 099551cb32..70273feb61 100644 --- a/bin/maintenance +++ b/bin/maintenance @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php maintenance $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php maintenance $@ diff --git a/bin/migrate b/bin/migrate index 28ebbd19e7..32bf7ee2f4 100755 --- a/bin/migrate +++ b/bin/migrate @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php migrate $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php migrate $@ diff --git a/bin/queue-count-failed b/bin/queue-count-failed index ca8f2b4291..93a6b3136c 100644 --- a/bin/queue-count-failed +++ b/bin/queue-count-failed @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php queue-count --type=failed $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php queue-count --type=failed $@ diff --git a/bin/queue-count-processing b/bin/queue-count-processing index 325d86111d..18e89664bd 100644 --- a/bin/queue-count-processing +++ b/bin/queue-count-processing @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php queue-count --type=processing $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php queue-count --type=processing $@ diff --git a/bin/queue-count-success b/bin/queue-count-success index 34fc54b4c1..b38bfa2159 100644 --- a/bin/queue-count-success +++ b/bin/queue-count-success @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php queue-count --type=success $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php queue-count --type=success $@ diff --git a/bin/queue-retry b/bin/queue-retry index f9473e6b07..2224a66b3b 100644 --- a/bin/queue-retry +++ b/bin/queue-retry @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php queue-retry $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php queue-retry $@ diff --git a/bin/realtime b/bin/realtime index e43dc269e0..debd4baf2a 100644 --- a/bin/realtime +++ b/bin/realtime @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/realtime.php $@ \ No newline at end of file +exec php /usr/src/code/app/realtime.php $@ diff --git a/bin/schedule-executions b/bin/schedule-executions index f239cad206..5d503e374c 100644 --- a/bin/schedule-executions +++ b/bin/schedule-executions @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php schedule-executions $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php schedule-executions $@ diff --git a/bin/schedule-functions b/bin/schedule-functions index 10edbe8226..beca1a0420 100644 --- a/bin/schedule-functions +++ b/bin/schedule-functions @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php schedule-functions $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php schedule-functions $@ diff --git a/bin/schedule-messages b/bin/schedule-messages index fa7219f6ea..3f17a279b5 100644 --- a/bin/schedule-messages +++ b/bin/schedule-messages @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php schedule-messages $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php schedule-messages $@ diff --git a/bin/sdks b/bin/sdks index ab73414829..3180813ea1 100644 --- a/bin/sdks +++ b/bin/sdks @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php sdks $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php sdks $@ diff --git a/bin/specs b/bin/specs index e77d1487d4..52875a1675 100644 --- a/bin/specs +++ b/bin/specs @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php specs $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php specs $@ diff --git a/bin/ssl b/bin/ssl index 83dcf6a026..0cc12375d0 100755 --- a/bin/ssl +++ b/bin/ssl @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php ssl $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php ssl $@ diff --git a/bin/stats-resources b/bin/stats-resources index 3104bab896..9cc67fb4a6 100644 --- a/bin/stats-resources +++ b/bin/stats-resources @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php stats-resources $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php stats-resources $@ diff --git a/bin/test b/bin/test index a2153fc536..c3b0d9b74c 100755 --- a/bin/test +++ b/bin/test @@ -1,3 +1,3 @@ #!/bin/sh -/usr/src/code/vendor/bin/phpunit --configuration /usr/src/code/phpunit.xml $@ \ No newline at end of file +exec /usr/src/code/vendor/bin/phpunit --configuration /usr/src/code/phpunit.xml $@ diff --git a/bin/upgrade b/bin/upgrade index ce32b9ca30..fd1f35b65a 100755 --- a/bin/upgrade +++ b/bin/upgrade @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php upgrade $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php upgrade $@ diff --git a/bin/vars b/bin/vars index 19e3f1ebf2..238d004675 100644 --- a/bin/vars +++ b/bin/vars @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php vars $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php vars $@ diff --git a/bin/worker-audits b/bin/worker-audits index 3df65d65e8..9bf81b93e2 100644 --- a/bin/worker-audits +++ b/bin/worker-audits @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php audits $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php audits $@ diff --git a/bin/worker-builds b/bin/worker-builds index 3400111cb5..7ddf5792de 100644 --- a/bin/worker-builds +++ b/bin/worker-builds @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php builds $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php builds $@ diff --git a/bin/worker-certificates b/bin/worker-certificates index 901688c4c8..958627bc33 100755 --- a/bin/worker-certificates +++ b/bin/worker-certificates @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php certificates $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php certificates $@ diff --git a/bin/worker-databases b/bin/worker-databases index 61e09aa9f1..235c16927c 100644 --- a/bin/worker-databases +++ b/bin/worker-databases @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php databases $@ +exec php /usr/src/code/app/worker.php databases $@ diff --git a/bin/worker-deletes b/bin/worker-deletes index 7c9793e6cb..c9711a4e66 100644 --- a/bin/worker-deletes +++ b/bin/worker-deletes @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php deletes $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php deletes $@ diff --git a/bin/worker-functions b/bin/worker-functions index 4757b1b72a..e82506d0e2 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php functions $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php functions $@ diff --git a/bin/worker-mails b/bin/worker-mails index fee8a96da7..ffb4f7ea77 100644 --- a/bin/worker-mails +++ b/bin/worker-mails @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php mails $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php mails $@ diff --git a/bin/worker-messaging b/bin/worker-messaging index e6edf80f06..6f39348419 100644 --- a/bin/worker-messaging +++ b/bin/worker-messaging @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php messaging $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php messaging $@ diff --git a/bin/worker-migrations b/bin/worker-migrations index 32d4aef468..2728d9d22c 100644 --- a/bin/worker-migrations +++ b/bin/worker-migrations @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php migrations $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php migrations $@ diff --git a/bin/worker-stats-resources b/bin/worker-stats-resources index 9c5d2bebff..70bfacbe52 100644 --- a/bin/worker-stats-resources +++ b/bin/worker-stats-resources @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php stats-resources $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php stats-resources $@ diff --git a/bin/worker-stats-usage b/bin/worker-stats-usage index 2c267d805e..e6dd849a48 100644 --- a/bin/worker-stats-usage +++ b/bin/worker-stats-usage @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php stats-usage $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php stats-usage $@ diff --git a/bin/worker-webhooks b/bin/worker-webhooks index 93f8027a81..63fc13e3c5 100644 --- a/bin/worker-webhooks +++ b/bin/worker-webhooks @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/worker.php webhooks $@ \ No newline at end of file +exec php /usr/src/code/app/worker.php webhooks $@ From d98c06abf2fbae4bfb7bd661aacf51cb671e9e74 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Sun, 25 May 2025 17:48:09 +0200 Subject: [PATCH 03/17] feat(worker): install stop hooks and close consumer --- app/worker.php | 13 +++++++++++-- bin/screenshot | 2 +- composer.lock | 12 ++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/app/worker.php b/app/worker.php index 597e8a9943..b5b65890ea 100644 --- a/app/worker.php +++ b/app/worker.php @@ -18,7 +18,9 @@ use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Executor\Executor; +use Swoole\Process; use Swoole\Runtime; +use Swoole\Timer; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Pool as CachePool; use Utopia\Cache\Adapter\Sharding; @@ -480,8 +482,15 @@ $worker }); $worker->workerStart() - ->action(function () use ($workerName) { - Console::info("Worker $workerName started"); + ->action(function () use ($worker, $workerName) { + Console::info("Worker $workerName started"); + + Process::signal(SIGTERM, function () use ($worker, $workerName) { + Console::info("Stopping worker $workerName."); + + $worker->stop(); + Timer::clearAll(); + }); }); $worker->start(); diff --git a/bin/screenshot b/bin/screenshot index 4d8ceb998f..aef15eb96f 100755 --- a/bin/screenshot +++ b/bin/screenshot @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php screenshot $@ \ No newline at end of file +exec php /usr/src/code/app/cli.php screenshot $@ \ No newline at end of file diff --git a/composer.lock b/composer.lock index b7b1f23545..8603361673 100644 --- a/composer.lock +++ b/composer.lock @@ -4264,16 +4264,16 @@ }, { "name": "utopia-php/queue", - "version": "0.11.0", + "version": "0.11.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "06b5ced0eaed2ecc6aab6d8e1b4d96bff37a1ce5" + "reference": "498bbbef418b1db71b51e1bb62f5d1d752ddd8d6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/06b5ced0eaed2ecc6aab6d8e1b4d96bff37a1ce5", - "reference": "06b5ced0eaed2ecc6aab6d8e1b4d96bff37a1ce5", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/498bbbef418b1db71b51e1bb62f5d1d752ddd8d6", + "reference": "498bbbef418b1db71b51e1bb62f5d1d752ddd8d6", "shasum": "" }, "require": { @@ -4324,9 +4324,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.11.0" + "source": "https://github.com/utopia-php/queue/tree/0.11.1" }, - "time": "2025-05-30T09:52:38+00:00" + "time": "2025-05-30T11:50:34+00:00" }, { "name": "utopia-php/registry", From 0ac6decc99829b09cc93a9d50cbe3587477face4 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 2 Jun 2025 14:04:03 +0530 Subject: [PATCH 04/17] Fix github comments and deployment creation on branch deletion --- app/controllers/api/vcs.php | 5 +- composer.lock | 108 +++++++++++++++++------------------ docker-compose.yml | 2 +- src/Appwrite/Vcs/Comment.php | 16 ++++-- 4 files changed, 66 insertions(+), 65 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 588ca417cf..c8a0a36a85 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -1215,6 +1215,7 @@ App::post('/v1/vcs/github/events') if ($event == $github::EVENT_PUSH) { $providerBranchCreated = $parsedPayload["branchCreated"] ?? false; + $providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false; $providerBranch = $parsedPayload["branch"] ?? ''; $providerBranchUrl = $parsedPayload["branchUrl"] ?? ''; $providerRepositoryId = $parsedPayload["repositoryId"] ?? ''; @@ -1236,8 +1237,8 @@ App::post('/v1/vcs/github/events') Query::limit(100), ])); - // create new deployment only on push and not when branch is created - if (!$providerBranchCreated) { + // create new deployment only on push and not when branch is created or deleted + if (!$providerBranchCreated && !$providerBranchDeleted) { $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', false, $dbForPlatform, $queueForBuilds, $getProjectDB, $request); } } elseif ($event == $github::EVENT_INSTALLATION) { diff --git a/composer.lock b/composer.lock index 729174e86a..8c8bcc0a6e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3230eea9e592f0b712e78c1ef5f5e244", + "content-hash": "d1e8bc951ce7d70a3254381a9f639971", "packages": [ { "name": "adhocore/jwt", @@ -283,16 +283,16 @@ }, { "name": "brick/math", - "version": "0.12.3", + "version": "0.13.1", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba" + "reference": "fc7ed316430118cc7836bf45faff18d5dfc8de04" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/866551da34e9a618e64a819ee1e01c20d8a588ba", - "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba", + "url": "https://api.github.com/repos/brick/math/zipball/fc7ed316430118cc7836bf45faff18d5dfc8de04", + "reference": "fc7ed316430118cc7836bf45faff18d5dfc8de04", "shasum": "" }, "require": { @@ -331,7 +331,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.3" + "source": "https://github.com/brick/math/tree/0.13.1" }, "funding": [ { @@ -339,7 +339,7 @@ "type": "github" } ], - "time": "2025-02-28T13:11:00+00:00" + "time": "2025-03-29T13:50:30+00:00" }, { "name": "chillerlan/php-qrcode", @@ -2323,20 +2323,20 @@ }, { "name": "ramsey/uuid", - "version": "4.7.6", + "version": "4.8.1", "source": { "type": "git", "url": "https://github.com/ramsey/uuid.git", - "reference": "91039bc1faa45ba123c4328958e620d382ec7088" + "reference": "fdf4dd4e2ff1813111bd0ad58d7a1ddbb5b56c28" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/uuid/zipball/91039bc1faa45ba123c4328958e620d382ec7088", - "reference": "91039bc1faa45ba123c4328958e620d382ec7088", + "url": "https://api.github.com/repos/ramsey/uuid/zipball/fdf4dd4e2ff1813111bd0ad58d7a1ddbb5b56c28", + "reference": "fdf4dd4e2ff1813111bd0ad58d7a1ddbb5b56c28", "shasum": "" }, "require": { - "brick/math": "^0.8.8 || ^0.9 || ^0.10 || ^0.11 || ^0.12", + "brick/math": "^0.8.8 || ^0.9 || ^0.10 || ^0.11 || ^0.12 || ^0.13", "ext-json": "*", "php": "^8.0", "ramsey/collection": "^1.2 || ^2.0" @@ -2345,26 +2345,23 @@ "rhumsaa/uuid": "self.version" }, "require-dev": { - "captainhook/captainhook": "^5.10", + "captainhook/captainhook": "^5.25", "captainhook/plugin-composer": "^5.3", - "dealerdirect/phpcodesniffer-composer-installer": "^0.7.0", - "doctrine/annotations": "^1.8", - "ergebnis/composer-normalize": "^2.15", - "mockery/mockery": "^1.3", + "dealerdirect/phpcodesniffer-composer-installer": "^1.0", + "ergebnis/composer-normalize": "^2.47", + "mockery/mockery": "^1.6", "paragonie/random-lib": "^2", - "php-mock/php-mock": "^2.2", - "php-mock/php-mock-mockery": "^1.3", - "php-parallel-lint/php-parallel-lint": "^1.1", - "phpbench/phpbench": "^1.0", - "phpstan/extension-installer": "^1.1", - "phpstan/phpstan": "^1.8", - "phpstan/phpstan-mockery": "^1.1", - "phpstan/phpstan-phpunit": "^1.1", - "phpunit/phpunit": "^8.5 || ^9", - "ramsey/composer-repl": "^1.4", - "slevomat/coding-standard": "^8.4", - "squizlabs/php_codesniffer": "^3.5", - "vimeo/psalm": "^4.9" + "php-mock/php-mock": "^2.6", + "php-mock/php-mock-mockery": "^1.5", + "php-parallel-lint/php-parallel-lint": "^1.4.0", + "phpbench/phpbench": "^1.2.14", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^9.6", + "slevomat/coding-standard": "^8.18", + "squizlabs/php_codesniffer": "^3.13" }, "suggest": { "ext-bcmath": "Enables faster math with arbitrary-precision integers using BCMath.", @@ -2399,19 +2396,9 @@ ], "support": { "issues": "https://github.com/ramsey/uuid/issues", - "source": "https://github.com/ramsey/uuid/tree/4.7.6" + "source": "https://github.com/ramsey/uuid/tree/4.8.1" }, - "funding": [ - { - "url": "https://github.com/ramsey", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/ramsey/uuid", - "type": "tidelift" - } - ], - "time": "2024-04-27T21:32:50+00:00" + "time": "2025-06-01T06:28:46+00:00" }, { "name": "spomky-labs/otphp", @@ -4597,16 +4584,16 @@ }, { "name": "utopia-php/vcs", - "version": "0.10.2", + "version": "dev-feat-check-deleted-attribute-on-push", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "1f9823ebcb8fd098607de0074f18f48e28985012" + "reference": "ba43fd0e797a61c92197885413b650aed6a45577" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/1f9823ebcb8fd098607de0074f18f48e28985012", - "reference": "1f9823ebcb8fd098607de0074f18f48e28985012", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/ba43fd0e797a61c92197885413b650aed6a45577", + "reference": "ba43fd0e797a61c92197885413b650aed6a45577", "shasum": "" }, "require": { @@ -4640,9 +4627,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/0.10.2" + "source": "https://github.com/utopia-php/vcs/tree/feat-check-deleted-attribute-on-push" }, - "time": "2025-04-17T04:35:25+00:00" + "time": "2025-06-02T08:03:56+00:00" }, { "name": "utopia-php/websocket", @@ -5344,16 +5331,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.4.0", + "version": "v5.5.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "447a020a1f875a434d62f2a401f53b82a396e494" + "reference": "ae59794362fe85e051a58ad36b289443f57be7a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/447a020a1f875a434d62f2a401f53b82a396e494", - "reference": "447a020a1f875a434d62f2a401f53b82a396e494", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/ae59794362fe85e051a58ad36b289443f57be7a9", + "reference": "ae59794362fe85e051a58ad36b289443f57be7a9", "shasum": "" }, "require": { @@ -5396,9 +5383,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.4.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.5.0" }, - "time": "2024-12-30T11:07:19+00:00" + "time": "2025-05-31T08:24:38+00:00" }, { "name": "phar-io/manifest", @@ -8245,9 +8232,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/vcs", + "version": "dev-feat-check-deleted-attribute-on-push", + "alias": "0.10.99", + "alias_normalized": "0.10.99.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/vcs": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index cefd082c2f..29a43aca91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -213,7 +213,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:6.0.13 + image: appwrite/console:6.0.32 restart: unless-stopped networks: - appwrite diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 7c550ad528..3fc4e287f8 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -105,8 +105,10 @@ class Comment foreach ($project['site'] as $siteId => $site) { $extension = $site['status'] === 'building' ? 'gif' : 'png'; - $pathLight = '/images/vcs/status-' . $site['status'] . '-light.' . $extension; - $pathDark = '/images/vcs/status-' . $site['status'] . '-dark.' . $extension; + $imageStatus = in_array($site['status'], ['processing', 'building']) ? 'building' : $site['status']; + + $pathLight = '/images/vcs/status-' . $imageStatus . '-light.' . $extension; + $pathDark = '/images/vcs/status-' . $imageStatus . '-dark.' . $extension; $status = match ($site['status']) { 'waiting' => $this->generatImage($pathLight, $pathDark, 'Queued', 85) . ' _Queued_', @@ -149,10 +151,11 @@ class Comment $text .= "| :- | :- | :- | :- |\n"; foreach ($project['function'] as $functionId => $function) { - $extension = $function['status'] === 'building' ? 'gif' : 'png'; + $imageStatus = in_array($function['status'], ['processing', 'building']) ? 'building' : $function['status']; + $extension = $imageStatus === 'building' ? 'gif' : 'png'; - $pathLight = '/images/vcs/status-' . $function['status'] . '-light.' . $extension; - $pathDark = '/images/vcs/status-' . $function['status'] . '-dark.' . $extension; + $pathLight = '/images/vcs/status-' . $imageStatus . '-light.' . $extension; + $pathDark = '/images/vcs/status-' . $imageStatus . '-dark.' . $extension; $status = match ($function['status']) { 'waiting' => $this->generatImage($pathLight, $pathDark, 'Queued', 85) . ' _Queued_', @@ -168,7 +171,8 @@ class Comment $action = '[Authorize](' . $function['action']['url'] . ')'; } - $text .= "|  **{$function['name']}**
`$functionId`"; + $text .= "|  **{$function['name']}**"; + $text .= "| `{$functionId}`"; $text .= "| {$status}"; $text .= "| {$action}"; $text .= "|\n"; From e7963e68257cca174d7eb0f04def3823e2dad534 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 2 Jun 2025 14:07:53 +0530 Subject: [PATCH 05/17] Update composer.lock --- composer.lock | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/composer.lock b/composer.lock index 8c8bcc0a6e..fb9db2aa20 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d1e8bc951ce7d70a3254381a9f639971", + "content-hash": "3230eea9e592f0b712e78c1ef5f5e244", "packages": [ { "name": "adhocore/jwt", @@ -4584,16 +4584,16 @@ }, { "name": "utopia-php/vcs", - "version": "dev-feat-check-deleted-attribute-on-push", + "version": "0.10.3", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "ba43fd0e797a61c92197885413b650aed6a45577" + "reference": "a51ca929aa8a82cc790c2df8b653d6c60cb889d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/ba43fd0e797a61c92197885413b650aed6a45577", - "reference": "ba43fd0e797a61c92197885413b650aed6a45577", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/a51ca929aa8a82cc790c2df8b653d6c60cb889d3", + "reference": "a51ca929aa8a82cc790c2df8b653d6c60cb889d3", "shasum": "" }, "require": { @@ -4627,9 +4627,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/feat-check-deleted-attribute-on-push" + "source": "https://github.com/utopia-php/vcs/tree/0.10.3" }, - "time": "2025-06-02T08:03:56+00:00" + "time": "2025-06-02T08:23:44+00:00" }, { "name": "utopia-php/websocket", @@ -8232,18 +8232,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/vcs", - "version": "dev-feat-check-deleted-attribute-on-push", - "alias": "0.10.99", - "alias_normalized": "0.10.99.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/vcs": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 288f3b2cc83083241f359f1aab0a0af32fc0d789 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 2 Jun 2025 14:12:25 +0530 Subject: [PATCH 06/17] Fix order --- src/Appwrite/Vcs/Comment.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 3fc4e287f8..d1d50c2ffd 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -103,10 +103,10 @@ class Comment $text .= "| :- | :- | :- | :- | :- |\n"; foreach ($project['site'] as $siteId => $site) { - $extension = $site['status'] === 'building' ? 'gif' : 'png'; - $imageStatus = in_array($site['status'], ['processing', 'building']) ? 'building' : $site['status']; + $extension = $site['status'] === 'building' ? 'gif' : 'png'; + $pathLight = '/images/vcs/status-' . $imageStatus . '-light.' . $extension; $pathDark = '/images/vcs/status-' . $imageStatus . '-dark.' . $extension; From 7049a58f212b1a7a8b277905855244bee82cf634 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 2 Jun 2025 15:06:00 +0530 Subject: [PATCH 07/17] Update composer.lock --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index fb9db2aa20..9e757f9db8 100644 --- a/composer.lock +++ b/composer.lock @@ -4584,16 +4584,16 @@ }, { "name": "utopia-php/vcs", - "version": "0.10.3", + "version": "0.10.4", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "a51ca929aa8a82cc790c2df8b653d6c60cb889d3" + "reference": "f635b368909eb3c3fe57344fe43525e74e8fdc03" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/a51ca929aa8a82cc790c2df8b653d6c60cb889d3", - "reference": "a51ca929aa8a82cc790c2df8b653d6c60cb889d3", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/f635b368909eb3c3fe57344fe43525e74e8fdc03", + "reference": "f635b368909eb3c3fe57344fe43525e74e8fdc03", "shasum": "" }, "require": { @@ -4627,9 +4627,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/0.10.3" + "source": "https://github.com/utopia-php/vcs/tree/0.10.4" }, - "time": "2025-06-02T08:23:44+00:00" + "time": "2025-06-02T09:18:36+00:00" }, { "name": "utopia-php/websocket", From c92a14d31805b693de8046dc06bfb6f368c8b80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 2 Jun 2025 12:47:13 +0200 Subject: [PATCH 08/17] Simplify router redirect --- app/controllers/general.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index bff701792e..5beac9e1e8 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -749,12 +749,6 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw return false; } elseif ($type === 'redirect') { $url = $rule->getAttribute('redirectUrl', ''); - - $query = ($swooleRequest->server['query_string'] ?? ''); - if (!empty($query)) { - $url .= '?' . $query; - } - $response->redirect($url, \intval($rule->getAttribute('redirectStatusCode', 301))); return true; } else { From c449144926aa6280f358be9a56ac16e628f626d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 2 Jun 2025 17:56:13 +0200 Subject: [PATCH 09/17] Fix flaky usage tests --- tests/e2e/General/UsageTest.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 5549ef800d..f0bb32156d 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -183,9 +183,11 @@ class UsageTest extends Scope /** * @depends testPrepareUsersStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testUsersStats(array $data): array { + \sleep(1); // To prevent all 10 retries from running instantly + $requestsTotal = $data['requestsTotal']; $response = $this->client->call( @@ -362,6 +364,8 @@ class UsageTest extends Scope #[Retry(count: 10)] public function testStorageStats(array $data): array { + \sleep(1); // To prevent all 10 retries from running instantly + $bucketId = $data['bucketId']; $bucketsTotal = $data['bucketsTotal']; $requestsTotal = $data['requestsTotal']; @@ -577,9 +581,11 @@ class UsageTest extends Scope } /** @depends testPrepareDatabaseStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testDatabaseStats(array $data): array { + \sleep(1); // To prevent all 10 retries from running instantly + $databaseId = $data['databaseId']; $collectionId = $data['collectionId']; $requestsTotal = $data['requestsTotal']; @@ -799,9 +805,11 @@ class UsageTest extends Scope } /** @depends testPrepareFunctionsStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testFunctionsStats(array $data): array { + \sleep(1); // To prevent all 10 retries from running instantly + $functionId = $data['functionId']; $executionTime = $data['executionTime']; $executions = $data['executions']; @@ -927,9 +935,11 @@ class UsageTest extends Scope } /** @depends testPrepareSitesStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testSitesStats(array $data) { + \sleep(1); // To prevent all 10 retries from running instantly + $siteId = $data['siteId']; $executionTime = $data['executionTime'] ?? 0; $executions = $data['executions'] ?? 0; From 854304dd1643a1763b44a31b65aaab099078bc0c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 2 Jun 2025 17:09:03 +0100 Subject: [PATCH 10/17] tests: fix flaky account tests commit-id:3865765d --- tests/e2e/Scopes/Scope.php | 14 +++++++++ .../Account/AccountCustomClientTest.php | 30 ++++++++----------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/e2e/Scopes/Scope.php b/tests/e2e/Scopes/Scope.php index 6deaa62c05..2fa6416bf7 100644 --- a/tests/e2e/Scopes/Scope.php +++ b/tests/e2e/Scopes/Scope.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Scopes; +use Appwrite\Tests\Async; use Appwrite\Tests\Retryable; use PHPUnit\Framework\TestCase; use Tests\E2E\Client; @@ -10,6 +11,7 @@ use Utopia\Database\Helpers\ID; abstract class Scope extends TestCase { use Retryable; + use Async; protected ?Client $client = null; protected string $endpoint = 'http://localhost/v1'; @@ -43,6 +45,18 @@ abstract class Scope extends TestCase return []; } + protected function assertLastRequest(callable $probe, $timeoutMs = 20_000, $waitMs = 500): array + { + $this->assertEventually(function () use (&$request, $probe) { + $request = json_decode(file_get_contents('http://request-catcher:5000/__last_request__'), true); + $request['data'] = json_decode($request['data'], true); + + call_user_func($probe, $request); + }, $timeoutMs, $waitMs); + + return $request; + } + protected function getLastRequest(): array { sleep(2); diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 452b725c11..0cc2eb893a 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2034,7 +2034,6 @@ class AccountCustomClientTest extends Scope $this->assertEquals($response['body']['users'][0]['email'], $email); } - #[Retry(count: 2)] public function testCreatePhone(): array { $number = '+123456789'; @@ -2058,17 +2057,15 @@ class AccountCustomClientTest extends Scope $userId = $response['body']['userId']; - \sleep(7); - - $smsRequest = $this->getLastRequest(); - - $this->assertEquals('http://request-catcher:5000/mock-sms', $smsRequest['url']); - $this->assertEquals('Appwrite Mock Message Sender', $smsRequest['headers']['User-Agent']); - $this->assertEquals('username', $smsRequest['headers']['X-Username']); - $this->assertEquals('password', $smsRequest['headers']['X-Key']); - $this->assertEquals('POST', $smsRequest['method']); - $this->assertEquals('+123456789', $smsRequest['data']['from']); - $this->assertEquals($number, $smsRequest['data']['to']); + $smsRequest = $this->assertLastRequest(function (array $request) use ($number) { + $this->assertEquals('http://request-catcher:5000/mock-sms', $request['url']); + $this->assertEquals('Appwrite Mock Message Sender', $request['headers']['User-Agent']); + $this->assertEquals('username', $request['headers']['X-Username']); + $this->assertEquals('password', $request['headers']['X-Key']); + $this->assertEquals('POST', $request['method']); + $this->assertEquals('+123456789', $request['data']['from']); + $this->assertEquals($number, $request['data']['to']); + }); $data['token'] = $smsRequest['data']['message']; $data['id'] = $userId; @@ -2396,7 +2393,6 @@ class AccountCustomClientTest extends Scope /** * @depends testUpdatePhone */ - #[Retry(count: 3)] public function testPhoneVerification(array $data): array { $session = $data['session'] ?? ''; @@ -2416,10 +2412,10 @@ class AccountCustomClientTest extends Scope $this->assertEmpty($response['body']['secret']); $this->assertTrue((new DatetimeValidator())->isValid($response['body']['expire'])); - $smsRequest = $this->getLastRequest(); - - $message = $smsRequest['data']['message']; - $token = substr($message, 0, 6); + $smsRequest = $this->assertLastRequest(function ($request) { + $this->assertArrayHasKey('data', $request); + $this->assertArrayHasKey('message', $request['data']); + }); /** * Test for FAILURE From b9579c8922eede87d5a398af5392a0321ba2be9c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 2 Jun 2025 18:36:48 +0100 Subject: [PATCH 11/17] tests: fix flaky messaging test commit-id:336a4f07 --- .../Messaging/MessagingConsoleClientTest.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php index 245eb3e8de..49d13128e2 100644 --- a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php +++ b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php @@ -206,15 +206,17 @@ class MessagingConsoleClientTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - $logs = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topic['body']['$id'] . '/logs', \array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + $this->assertEventually(function () use ($topic) { + $logs = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topic['body']['$id'] . '/logs', \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); - $this->assertEquals($logs['headers']['status-code'], 200); - $this->assertIsArray($logs['body']['logs']); - $this->assertCount(2, $logs['body']['logs']); - $this->assertIsNumeric($logs['body']['total']); + $this->assertEquals($logs['headers']['status-code'], 200); + $this->assertIsArray($logs['body']['logs']); + $this->assertCount(2, $logs['body']['logs']); + $this->assertIsNumeric($logs['body']['total']); + }); $logs = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topic['body']['$id'] . '/logs', \array_merge([ 'content-type' => 'application/json', From c19d123d48f0f83353845e2600495287550951f0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 2 Jun 2025 18:20:08 +0100 Subject: [PATCH 12/17] tests: make usagetests robust commit-id:35fcc17d --- tests/e2e/General/UsageTest.php | 506 ++++++++++++++++---------------- 1 file changed, 258 insertions(+), 248 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index f0bb32156d..6ff9dafada 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -3,7 +3,6 @@ namespace Tests\E2E\General; use Appwrite\Platform\Modules\Compute\Specification; -use Appwrite\Tests\Retry; use CURLFile; use DateTime; use Tests\E2E\Client; @@ -183,42 +182,43 @@ class UsageTest extends Scope /** * @depends testPrepareUsersStats */ - #[Retry(count: 10)] public function testUsersStats(array $data): array { - \sleep(1); // To prevent all 10 retries from running instantly - $requestsTotal = $data['requestsTotal']; - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); + $this->assertEventually(function () { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertGreaterThanOrEqual(31, count($response['body'])); - $this->validateDates($response['body']['network']); - $this->validateDates($response['body']['requests']); - $this->validateDates($response['body']['users']); - $this->assertArrayHasKey('executionsBreakdown', $response['body']); - $this->assertArrayHasKey('bucketsBreakdown', $response['body']); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertGreaterThanOrEqual(31, count($response['body'])); + $this->validateDates($response['body']['network']); + $this->validateDates($response['body']['requests']); + $this->validateDates($response['body']['users']); + $this->assertArrayHasKey('executionsBreakdown', $response['body']); + $this->assertArrayHasKey('bucketsBreakdown', $response['body']); + }); - $response = $this->client->call( - Client::METHOD_GET, - '/users/usage?range=90d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () { + $response = $this->client->call( + Client::METHOD_GET, + '/users/usage?range=90d', + $this->getConsoleHeaders() + ); - $this->assertEquals('90d', $response['body']['range']); - $this->assertEquals(90, count($response['body']['users'])); - $this->assertEquals(90, count($response['body']['sessions'])); - $this->assertEquals((self::CREATE / 2), $response['body']['users'][array_key_last($response['body']['users'])]['value']); + $this->assertEquals('90d', $response['body']['range']); + $this->assertEquals(90, count($response['body']['users'])); + $this->assertEquals(90, count($response['body']['sessions'])); + $this->assertEquals((self::CREATE / 2), $response['body']['users'][array_key_last($response['body']['users'])]['value']); + }); return array_merge($data, [ 'requestsTotal' => $requestsTotal @@ -361,55 +361,58 @@ class UsageTest extends Scope /** * @depends testPrepareStorageStats */ - #[Retry(count: 10)] public function testStorageStats(array $data): array { - \sleep(1); // To prevent all 10 retries from running instantly - $bucketId = $data['bucketId']; $bucketsTotal = $data['bucketsTotal']; $requestsTotal = $data['requestsTotal']; $storageTotal = $data['storageTotal']; $filesTotal = $data['filesTotal']; - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1d', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); + $this->assertEventually(function () use ($requestsTotal, $storageTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1d', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); - $this->assertGreaterThanOrEqual(31, count($response['body'])); - $this->assertEquals(1, count($response['body']['requests'])); - $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); - $this->validateDates($response['body']['requests']); - $this->assertEquals($storageTotal, $response['body']['filesStorageTotal']); + $this->assertGreaterThanOrEqual(31, count($response['body'])); + $this->assertEquals(1, count($response['body']['requests'])); + $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); + $this->validateDates($response['body']['requests']); + $this->assertEquals($storageTotal, $response['body']['filesStorageTotal']); + }); - $response = $this->client->call( - Client::METHOD_GET, - '/storage/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () use ($bucketsTotal, $filesTotal, $storageTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/storage/usage?range=30d', + $this->getConsoleHeaders() + ); - $this->assertEquals($storageTotal, $response['body']['storage'][array_key_last($response['body']['storage'])]['value']); - $this->validateDates($response['body']['storage']); - $this->assertEquals($bucketsTotal, $response['body']['buckets'][array_key_last($response['body']['buckets'])]['value']); - $this->validateDates($response['body']['buckets']); - $this->assertEquals($filesTotal, $response['body']['files'][array_key_last($response['body']['files'])]['value']); - $this->validateDates($response['body']['files']); + $this->assertEquals($storageTotal, $response['body']['storage'][array_key_last($response['body']['storage'])]['value']); + $this->validateDates($response['body']['storage']); + $this->assertEquals($bucketsTotal, $response['body']['buckets'][array_key_last($response['body']['buckets'])]['value']); + $this->validateDates($response['body']['buckets']); + $this->assertEquals($filesTotal, $response['body']['files'][array_key_last($response['body']['files'])]['value']); + $this->validateDates($response['body']['files']); + }); - $response = $this->client->call( - Client::METHOD_GET, - '/storage/' . $bucketId . '/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () use ($bucketId, $storageTotal, $filesTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/storage/' . $bucketId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $this->assertEquals($storageTotal, $response['body']['storage'][array_key_last($response['body']['storage'])]['value']); - $this->assertEquals($filesTotal, $response['body']['files'][array_key_last($response['body']['files'])]['value']); + $this->assertEquals($storageTotal, $response['body']['storage'][array_key_last($response['body']['storage'])]['value']); + $this->assertEquals($filesTotal, $response['body']['files'][array_key_last($response['body']['files'])]['value']); + }); return $data; } @@ -581,11 +584,9 @@ class UsageTest extends Scope } /** @depends testPrepareDatabaseStats */ - #[Retry(count: 10)] + public function testDatabaseStats(array $data): array { - \sleep(1); // To prevent all 10 retries from running instantly - $databaseId = $data['databaseId']; $collectionId = $data['collectionId']; $requestsTotal = $data['requestsTotal']; @@ -593,60 +594,66 @@ class UsageTest extends Scope $collectionsTotal = $data['collectionsTotal']; $documentsTotal = $data['documentsTotal']; - sleep(self::WAIT); + $this->assertEventually(function () use ($requestsTotal, $databasesTotal, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1d', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1d', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); + $this->assertGreaterThanOrEqual(31, count($response['body'])); + $this->assertEquals(1, count($response['body']['requests'])); + $this->assertEquals(1, count($response['body']['network'])); + $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); + $this->validateDates($response['body']['requests']); + $this->assertEquals($databasesTotal, $response['body']['databasesTotal']); + $this->assertEquals($documentsTotal, $response['body']['documentsTotal']); + }); - $this->assertGreaterThanOrEqual(31, count($response['body'])); - $this->assertEquals(1, count($response['body']['requests'])); - $this->assertEquals(1, count($response['body']['network'])); - $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); - $this->validateDates($response['body']['requests']); - $this->assertEquals($databasesTotal, $response['body']['databasesTotal']); - $this->assertEquals($documentsTotal, $response['body']['documentsTotal']); + $this->assertEventually(function () use ($collectionsTotal, $databasesTotal, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/databases/usage?range=30d', + $this->getConsoleHeaders() + ); - $response = $this->client->call( - Client::METHOD_GET, - '/databases/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEquals($databasesTotal, $response['body']['databases'][array_key_last($response['body']['databases'])]['value']); + $this->validateDates($response['body']['databases']); + $this->assertEquals($collectionsTotal, $response['body']['collections'][array_key_last($response['body']['collections'])]['value']); + $this->validateDates($response['body']['collections']); + $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); + $this->validateDates($response['body']['documents']); + }); - $this->assertEquals($databasesTotal, $response['body']['databases'][array_key_last($response['body']['databases'])]['value']); - $this->validateDates($response['body']['databases']); - $this->assertEquals($collectionsTotal, $response['body']['collections'][array_key_last($response['body']['collections'])]['value']); - $this->validateDates($response['body']['collections']); - $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); - $this->validateDates($response['body']['documents']); + $this->assertEventually(function () use ($databaseId, $collectionsTotal, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/databases/' . $databaseId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $response = $this->client->call( - Client::METHOD_GET, - '/databases/' . $databaseId . '/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEquals($collectionsTotal, $response['body']['collections'][array_key_last($response['body']['collections'])]['value']); + $this->validateDates($response['body']['collections']); - $this->assertEquals($collectionsTotal, $response['body']['collections'][array_key_last($response['body']['collections'])]['value']); - $this->validateDates($response['body']['collections']); + $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); + $this->validateDates($response['body']['documents']); + }); - $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); - $this->validateDates($response['body']['documents']); + $this->assertEventually(function () use ($databaseId, $collectionId, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/databases/' . $databaseId . '/collections/' . $collectionId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $response = $this->client->call( - Client::METHOD_GET, - '/databases/' . $databaseId . '/collections/' . $collectionId . '/usage?range=30d', - $this->getConsoleHeaders() - ); - - $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); - $this->validateDates($response['body']['documents']); + $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); + $this->validateDates($response['body']['documents']); + }); return $data; } @@ -805,69 +812,69 @@ class UsageTest extends Scope } /** @depends testPrepareFunctionsStats */ - #[Retry(count: 10)] public function testFunctionsStats(array $data): array { - \sleep(1); // To prevent all 10 retries from running instantly - $functionId = $data['functionId']; $executionTime = $data['executionTime']; $executions = $data['executions']; - $response = $this->client->call( - Client::METHOD_GET, - '/functions/' . $functionId . '/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () use ($functionId, $executions, $executionTime) { + $response = $this->client->call( + Client::METHOD_GET, + '/functions/' . $functionId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(24, count($response['body'])); - $this->assertEquals('30d', $response['body']['range']); - $this->assertIsArray($response['body']['deployments']); - $this->assertIsArray($response['body']['deploymentsStorage']); - $this->assertIsNumeric($response['body']['deploymentsStorageTotal']); - $this->assertIsNumeric($response['body']['buildsMbSecondsTotal']); - $this->assertIsNumeric($response['body']['executionsMbSecondsTotal']); - $this->assertIsArray($response['body']['builds']); - $this->assertIsArray($response['body']['buildsTime']); - $this->assertIsArray($response['body']['buildsMbSeconds']); - $this->assertIsArray($response['body']['executions']); - $this->assertIsArray($response['body']['executionsTime']); - $this->assertIsArray($response['body']['executionsMbSeconds']); - $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); - $this->validateDates($response['body']['executions']); - $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); - $this->validateDates($response['body']['executionsTime']); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(24, count($response['body'])); + $this->assertEquals('30d', $response['body']['range']); + $this->assertIsArray($response['body']['deployments']); + $this->assertIsArray($response['body']['deploymentsStorage']); + $this->assertIsNumeric($response['body']['deploymentsStorageTotal']); + $this->assertIsNumeric($response['body']['buildsMbSecondsTotal']); + $this->assertIsNumeric($response['body']['executionsMbSecondsTotal']); + $this->assertIsArray($response['body']['builds']); + $this->assertIsArray($response['body']['buildsTime']); + $this->assertIsArray($response['body']['buildsMbSeconds']); + $this->assertIsArray($response['body']['executions']); + $this->assertIsArray($response['body']['executionsTime']); + $this->assertIsArray($response['body']['executionsMbSeconds']); + $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); + $this->validateDates($response['body']['executions']); + $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); + $this->validateDates($response['body']['executionsTime']); + }); - $response = $this->client->call( - Client::METHOD_GET, - '/functions/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () use ($executions, $executionTime) { + $response = $this->client->call( + Client::METHOD_GET, + '/functions/usage?range=30d', + $this->getConsoleHeaders() + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(25, count($response['body'])); - $this->assertEquals($response['body']['range'], '30d'); - $this->assertIsArray($response['body']['functions']); - $this->assertIsArray($response['body']['deployments']); - $this->assertIsArray($response['body']['deploymentsStorage']); - $this->assertIsArray($response['body']['builds']); - $this->assertIsArray($response['body']['buildsTime']); - $this->assertIsArray($response['body']['buildsMbSeconds']); - $this->assertIsArray($response['body']['executions']); - $this->assertIsArray($response['body']['executionsTime']); - $this->assertIsArray($response['body']['executionsMbSeconds']); - $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); - $this->validateDates($response['body']['executions']); - $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); - $this->validateDates($response['body']['executionsTime']); - $this->assertGreaterThan(0, $response['body']['buildsTime'][array_key_last($response['body']['buildsTime'])]['value']); - $this->validateDates($response['body']['buildsTime']); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(25, count($response['body'])); + $this->assertEquals($response['body']['range'], '30d'); + $this->assertIsArray($response['body']['functions']); + $this->assertIsArray($response['body']['deployments']); + $this->assertIsArray($response['body']['deploymentsStorage']); + $this->assertIsArray($response['body']['builds']); + $this->assertIsArray($response['body']['buildsTime']); + $this->assertIsArray($response['body']['buildsMbSeconds']); + $this->assertIsArray($response['body']['executions']); + $this->assertIsArray($response['body']['executionsTime']); + $this->assertIsArray($response['body']['executionsMbSeconds']); + $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); + $this->validateDates($response['body']['executions']); + $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); + $this->validateDates($response['body']['executionsTime']); + $this->assertGreaterThan(0, $response['body']['buildsTime'][array_key_last($response['body']['buildsTime'])]['value']); + $this->validateDates($response['body']['buildsTime']); + }); return $data; } - public function testPrepareSitesStats(): array { $siteId = $this->setupSite([ @@ -935,77 +942,79 @@ class UsageTest extends Scope } /** @depends testPrepareSitesStats */ - #[Retry(count: 10)] public function testSitesStats(array $data) { - \sleep(1); // To prevent all 10 retries from running instantly - $siteId = $data['siteId']; $executionTime = $data['executionTime'] ?? 0; $executions = $data['executions'] ?? 0; $deploymentsSuccess = $data['deploymentsSuccess']; $deploymentsFailed = $data['deploymentsFailed']; - $response = $this->client->call( - Client::METHOD_GET, - '/sites/' . $siteId . '/usage?range=30d', - $this->getConsoleHeaders() - ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(30, count($response['body'])); - $this->assertEquals('30d', $response['body']['range']); - $this->assertIsArray($response['body']['deployments']); - $this->assertEquals($deploymentsSuccess, $response['body']['buildsSuccessTotal']); - $this->assertEquals($deploymentsFailed, $response['body']['buildsFailedTotal']); - $this->assertIsArray($response['body']['deploymentsStorage']); - $this->assertIsNumeric($response['body']['deploymentsStorageTotal']); - $this->assertIsNumeric($response['body']['buildsMbSecondsTotal']); - $this->assertIsNumeric($response['body']['executionsMbSecondsTotal']); - $this->assertIsArray($response['body']['builds']); - $this->assertIsArray($response['body']['buildsTime']); - $this->assertIsArray($response['body']['buildsMbSeconds']); - $this->assertIsArray($response['body']['executions']); - $this->assertIsArray($response['body']['executionsTime']); - $this->assertIsArray($response['body']['executionsMbSeconds']); - $this->assertIsArray($response['body']['buildsSuccess']); - $this->assertIsArray($response['body']['buildsFailed']); - $this->assertIsArray($response['body']['requests']); - $this->assertIsArray($response['body']['inbound']); - $this->assertIsArray($response['body']['outbound']); - $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); - $this->validateDates($response['body']['executions']); - $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); - $this->validateDates($response['body']['executionsTime']); + $this->assertEventually(function () use ($siteId, $deploymentsSuccess, $deploymentsFailed, $executions, $executionTime) { + $response = $this->client->call( + Client::METHOD_GET, + '/sites/' . $siteId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $response = $this->client->call( - Client::METHOD_GET, - '/sites/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(30, count($response['body'])); + $this->assertEquals('30d', $response['body']['range']); + $this->assertIsArray($response['body']['deployments']); + $this->assertEquals($deploymentsSuccess, $response['body']['buildsSuccessTotal']); + $this->assertEquals($deploymentsFailed, $response['body']['buildsFailedTotal']); + $this->assertIsArray($response['body']['deploymentsStorage']); + $this->assertIsNumeric($response['body']['deploymentsStorageTotal']); + $this->assertIsNumeric($response['body']['buildsMbSecondsTotal']); + $this->assertIsNumeric($response['body']['executionsMbSecondsTotal']); + $this->assertIsArray($response['body']['builds']); + $this->assertIsArray($response['body']['buildsTime']); + $this->assertIsArray($response['body']['buildsMbSeconds']); + $this->assertIsArray($response['body']['executions']); + $this->assertIsArray($response['body']['executionsTime']); + $this->assertIsArray($response['body']['executionsMbSeconds']); + $this->assertIsArray($response['body']['buildsSuccess']); + $this->assertIsArray($response['body']['buildsFailed']); + $this->assertIsArray($response['body']['requests']); + $this->assertIsArray($response['body']['inbound']); + $this->assertIsArray($response['body']['outbound']); + $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); + $this->validateDates($response['body']['executions']); + $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); + $this->validateDates($response['body']['executionsTime']); + }); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(31, count($response['body'])); - $this->assertEquals($response['body']['range'], '30d'); - $this->assertIsArray($response['body']['sites']); - $this->assertIsArray($response['body']['deployments']); - $this->assertIsArray($response['body']['deploymentsStorage']); - $this->assertIsArray($response['body']['builds']); - $this->assertIsArray($response['body']['buildsTime']); - $this->assertIsArray($response['body']['buildsMbSeconds']); - $this->assertIsArray($response['body']['executions']); - $this->assertIsArray($response['body']['executionsTime']); - $this->assertIsArray($response['body']['executionsMbSeconds']); - $this->assertIsArray($response['body']['buildsSuccess']); - $this->assertIsArray($response['body']['buildsFailed']); - $this->assertIsArray($response['body']['requests']); - $this->assertIsArray($response['body']['inbound']); - $this->assertIsArray($response['body']['outbound']); - $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); - $this->validateDates($response['body']['executions']); - $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); - $this->validateDates($response['body']['executionsTime']); - $this->assertGreaterThan(0, $response['body']['buildsTime'][array_key_last($response['body']['buildsTime'])]['value']); - $this->validateDates($response['body']['buildsTime']); + $this->assertEventually(function () use ($executions, $executionTime) { + $response = $this->client->call( + Client::METHOD_GET, + '/sites/usage?range=30d', + $this->getConsoleHeaders() + ); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(31, count($response['body'])); + $this->assertEquals($response['body']['range'], '30d'); + $this->assertIsArray($response['body']['sites']); + $this->assertIsArray($response['body']['deployments']); + $this->assertIsArray($response['body']['deploymentsStorage']); + $this->assertIsArray($response['body']['builds']); + $this->assertIsArray($response['body']['buildsTime']); + $this->assertIsArray($response['body']['buildsMbSeconds']); + $this->assertIsArray($response['body']['executions']); + $this->assertIsArray($response['body']['executionsTime']); + $this->assertIsArray($response['body']['executionsMbSeconds']); + $this->assertIsArray($response['body']['buildsSuccess']); + $this->assertIsArray($response['body']['buildsFailed']); + $this->assertIsArray($response['body']['requests']); + $this->assertIsArray($response['body']['inbound']); + $this->assertIsArray($response['body']['outbound']); + $this->assertEquals($executions, $response['body']['executions'][array_key_last($response['body']['executions'])]['value']); + $this->validateDates($response['body']['executions']); + $this->assertEquals($executionTime, $response['body']['executionsTime'][array_key_last($response['body']['executionsTime'])]['value']); + $this->validateDates($response['body']['executionsTime']); + $this->assertGreaterThan(0, $response['body']['buildsTime'][array_key_last($response['body']['buildsTime'])]['value']); + $this->validateDates($response['body']['buildsTime']); + }); } /** @depends testFunctionsStats */ @@ -1042,30 +1051,33 @@ class UsageTest extends Scope $domain = $rule['body']['domain']; - $response = $this->client->call( - Client::METHOD_GET, - '/functions/' . $functionId . '/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () use (&$response, $functionId) { + $response = $this->client->call( + Client::METHOD_GET, + '/functions/' . $functionId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(24, count($response['body'])); - $this->assertEquals('30d', $response['body']['range']); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(24, count($response['body'])); + $this->assertEquals('30d', $response['body']['range']); + }); $functionsMetrics = $response['body']; - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEventually(function () use (&$response) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + $this->assertEquals(200, $response['headers']['status-code']); + }); $projectMetrics = $response['body']; @@ -1080,8 +1092,6 @@ class UsageTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - $tries = 0; - $this->assertEventually(function () use ($functionId, $functionsMetrics, $projectMetrics) { // Compare new values with old values $response = $this->client->call( From 71c54f5742196dd55aaa925ab6598fe34013dc14 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 2 Jun 2025 17:15:01 +0100 Subject: [PATCH 13/17] tests: increase deployment timeouts commit-id:7d7dffac --- tests/e2e/Services/Functions/FunctionsBase.php | 2 +- tests/e2e/Services/Migrations/MigrationsBase.php | 2 +- tests/e2e/Services/Sites/SitesBase.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsBase.php b/tests/e2e/Services/Functions/FunctionsBase.php index 475eb9b9b4..27b67d851d 100644 --- a/tests/e2e/Services/Functions/FunctionsBase.php +++ b/tests/e2e/Services/Functions/FunctionsBase.php @@ -49,7 +49,7 @@ trait FunctionsBase 'x-appwrite-key' => $this->getProject()['apiKey'], ])); $this->assertEquals('ready', $deployment['body']['status'], 'Deployment status is not ready, deployment: ' . json_encode($deployment['body'], JSON_PRETTY_PRINT)); - }, 50000, 500); + }, 100000, 500); // Not === so multipart/form-data works fine too if (($params['activate'] ?? false) == true) { diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index c241b38e3d..b8b9439e64 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -871,7 +871,7 @@ trait MigrationsBase $this->assertEquals(1, $deployments['body']['total']); $this->assertEquals('ready', $deployments['body']['deployments'][0]['status'], 'Deployment status is not ready, deployment: ' . json_encode($deployments['body']['deployments'][0], JSON_PRETTY_PRINT)); - }, 50000, 500); + }, 100000, 500); // Attempt execution $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', [ diff --git a/tests/e2e/Services/Sites/SitesBase.php b/tests/e2e/Services/Sites/SitesBase.php index 00edcc1b72..2054744863 100644 --- a/tests/e2e/Services/Sites/SitesBase.php +++ b/tests/e2e/Services/Sites/SitesBase.php @@ -265,7 +265,7 @@ trait SitesBase $this->assertEventually(function () use ($siteId, $deploymentId) { $deployment = $this->getDeployment($siteId, $deploymentId); $this->assertEquals('ready', $deployment['body']['status'], 'Deployment status is not ready, deployment: ' . json_encode($deployment['body'], JSON_PRETTY_PRINT)); - }, 100000, 500); + }, 150000, 500); $this->assertEventually(function () use ($siteId, $deploymentId) { $site = $this->getSite($siteId); From 584ca3065fa64437b78980d3879b417c845de53b Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Fri, 30 May 2025 16:38:54 +0200 Subject: [PATCH 14/17] fix(gif preview): enable resource limits --- app/config/storage/inputs.php | 1 + app/config/storage/outputs.php | 1 + app/config/storage/resource_limits.php | 6 ++++++ app/init/configs.php | 2 ++ composer.lock | 24 ++++++++++++------------ 5 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 app/config/storage/resource_limits.php diff --git a/app/config/storage/inputs.php b/app/config/storage/inputs.php index edcf667d86..a102dfdcf0 100644 --- a/app/config/storage/inputs.php +++ b/app/config/storage/inputs.php @@ -7,4 +7,5 @@ return [ "png" => "image/png", "heic" => "image/heic", "webp" => "image/webp", + "gif" => "image/gif", ]; diff --git a/app/config/storage/outputs.php b/app/config/storage/outputs.php index 519ff825fe..3e6fd45651 100644 --- a/app/config/storage/outputs.php +++ b/app/config/storage/outputs.php @@ -8,4 +8,5 @@ return [ "webp" => "image/webp", "heic" => "image/heic", "avif" => "image/avif", + "gif" => "image/gif", ]; diff --git a/app/config/storage/resource_limits.php b/app/config/storage/resource_limits.php new file mode 100644 index 0000000000..cfbcea5a47 --- /dev/null +++ b/app/config/storage/resource_limits.php @@ -0,0 +1,6 @@ + Date: Tue, 18 Mar 2025 12:29:32 +0100 Subject: [PATCH 15/17] chore(audits): return queue pre-fetch results --- src/Appwrite/Platform/Workers/Audits.php | 57 ++++++++++++------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 76309145b8..ce8dbedd73 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,13 +12,13 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Queue\Result\Commit; +use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + protected const int BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,9 +27,7 @@ class Audits extends Action protected function getBatchSize(): int { - return System::getEnv('_APP_ENV', 'development') === 'development' - ? self::BATCH_SIZE_DEVELOPMENT - : self::BATCH_SIZE_PRODUCTION; + return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); } public static function getName(): string @@ -57,13 +55,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return void + * @return Commit|NoCommit * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): void + public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit { $payload = $message->getPayload() ?? []; @@ -123,29 +121,32 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $shouldProcessBatch = \count($this->logs) >= $batchSize; - if (!$shouldProcessBatch && \count($this->logs) > 0) { + $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs['logs']), 0); + $shouldProcessBatch = $logCount >= $batchSize; + if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if ($shouldProcessBatch) { - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); - - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); - - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); - - unset($this->logs[$internalId]); - } - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - } finally { - $this->lastTriggeredTime = time(); - } + if (!$shouldProcessBatch) { + return new NoCommit(); } + + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); + + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); + + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); + + unset($this->logs[$internalId]); + } + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } + $this->lastTriggeredTime = time(); + return new Commit(); } } From 7522a3a56ef84d4cf8aa16a12e24142b3b0c4a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 3 Jun 2025 13:20:24 +0200 Subject: [PATCH 16/17] Fix multiregion view logs --- src/Appwrite/Vcs/Comment.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index d1d50c2ffd..21e64b4b24 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -87,7 +87,7 @@ class Comment $i = 0; foreach ($projects as $projectId => $project) { $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_DOMAIN'); + $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN')); $text .= "## {$project['name']}\n\n"; $text .= "Project ID: `{$projectId}`\n\n"; From f9be8dc966ec0f8e15d1ed04b145f587e43bef40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 3 Jun 2025 13:20:56 +0200 Subject: [PATCH 17/17] Missed spot from previous commit --- src/Appwrite/Vcs/Comment.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 21e64b4b24..5f528e660f 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -201,7 +201,7 @@ class Comment public function generatImage(string $pathLight, string $pathDark, string $alt, int $width): string { $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_DOMAIN'); + $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN')); $imageLight = $protocol . '://' . $hostname . $pathLight; $imageDark = $protocol . '://' . $hostname . $pathDark;