From 65dd5fe9b3f0e8a178effc388c9287b5157f7559 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:34:28 +0530 Subject: [PATCH 01/11] feat: compute usage for OPTIONS and 4XX errors --- app/controllers/general.php | 15 +++++++++++++-- app/controllers/shared/api.php | 1 + docker-compose.yml | 14 +++++++------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 6db4a8b28d..ae6db43c70 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -737,7 +737,8 @@ App::options() ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->inject('project') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { /* * Appwrite Router */ @@ -760,6 +761,16 @@ App::options() ->addHeader('Access-Control-Allow-Origin', $origin) ->addHeader('Access-Control-Allow-Credentials', 'true') ->noContent(); + + /** OPTIONS requests in utopia do not execute shutdown handlers, as a result we need to track the OPTIONS requests explicitly + * @see https://github.com/utopia-php/http/blob/0.33.16/src/App.php#L825-L855 + */ + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize()) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) + ->setProject($project) + ->trigger(); }); App::error() @@ -874,7 +885,7 @@ App::error() } } - if ($publish && $project->getId() !== 'console') { + if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; $file = $request->getFiles('file'); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 7f7b73ab0c..64a9980bcb 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -803,6 +803,7 @@ App::shutdown() } } + var_dump("in shutdown"); if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; diff --git a/docker-compose.yml b/docker-compose.yml index facf0e6db9..3316e7dcf8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-assistant: - container_name: appwrite-assistant - image: appwrite/assistant:0.7.0 - networks: - - appwrite - environment: - - _APP_ASSISTANT_OPENAI_API_KEY + # appwrite-assistant: + # container_name: appwrite-assistant + # image: appwrite/assistant:0.7.0 + # networks: + # - appwrite + # environment: + # - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor From 12b494982ba2b016081660197dd5c76557a88521 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:37:54 +0530 Subject: [PATCH 02/11] feat: compute usage for OPTIONS and 4XX errors --- app/controllers/general.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index ae6db43c70..65979d3475 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -885,7 +885,10 @@ App::error() } } - if ($project->getId() !== 'console') { + /** + * If its not a publishable error, track usage stats. Publishable errors are >= 500 or those explicitly marked as publish=true in errors.php + */ + if (!$publish && $project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; $file = $request->getFiles('file'); From 2359c1adb6734afd4088de7609fa58107a30d185 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:38:59 +0530 Subject: [PATCH 03/11] feat: enable assistant --- docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3316e7dcf8..facf0e6db9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - # appwrite-assistant: - # container_name: appwrite-assistant - # image: appwrite/assistant:0.7.0 - # networks: - # - appwrite - # environment: - # - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-assistant: + container_name: appwrite-assistant + image: appwrite/assistant:0.7.0 + networks: + - appwrite + environment: + - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor From 30327f86fd967f92486040deb5f929d4e95bdb2c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:39:18 +0530 Subject: [PATCH 04/11] feat: enable assistant --- app/controllers/shared/api.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 64a9980bcb..7f7b73ab0c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -803,7 +803,6 @@ App::shutdown() } } - var_dump("in shutdown"); if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; From cf41d44de4b299d2535a1098b6b769be5aac96a7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 14:47:57 +0530 Subject: [PATCH 05/11] fix: general tests --- src/Appwrite/Platform/Workers/StatsUsage.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index c4d8b0e8d2..a755f723a0 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -17,13 +17,20 @@ class StatsUsage extends Action private int $lastTriggeredTime = 0; private int $keys = 0; private const INFINITY_PERIOD = '_inf_'; - private const KEYS_THRESHOLD = 10000; + private const BATCH_SIZE_DEVELOPMENT = 1; + private const BATCH_SIZE_PRODUCTION = 10_000; public static function getName(): string { return 'stats-usage'; } + private function getBatchSize(): int + { + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; + } /** * @throws Exception */ @@ -86,7 +93,7 @@ class StatsUsage extends Action // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( - $this->keys >= self::KEYS_THRESHOLD || + $this->keys >= $this->getBatchSize() || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); From f682e4ac6f8f04cd4b9158633b5c4c04a895cf7e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 14:56:15 +0530 Subject: [PATCH 06/11] chore: remoove sleep() for usage stats, since batch size is now 1 --- tests/e2e/General/UsageTest.php | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c0d0c80eb1..3e22f827bd 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -134,8 +134,6 @@ class UsageTest extends Scope #[Retry(count: 1)] public function testUsersStats(array $data): array { - sleep(self::WAIT); - $requestsTotal = $data['requestsTotal']; $response = $this->client->call( @@ -318,8 +316,6 @@ class UsageTest extends Scope $storageTotal = $data['storageTotal']; $filesTotal = $data['filesTotal']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/project/usage', @@ -476,8 +472,6 @@ class UsageTest extends Scope $requestsTotal += 1; - sleep(self::WAIT); - for ($i = 0; $i < self::CREATE; $i++) { $name = uniqid() . ' collection'; @@ -539,8 +533,6 @@ class UsageTest extends Scope $collectionsTotal = $data['collectionsTotal']; $documentsTotal = $data['documentsTotal']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/project/usage', @@ -709,8 +701,6 @@ class UsageTest extends Scope // $this->assertEquals(201, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -752,7 +742,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -792,8 +781,6 @@ class UsageTest extends Scope // $this->assertEquals(204, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -835,7 +822,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -1027,8 +1013,6 @@ class UsageTest extends Scope $executionTime = $data['executionTime']; $executions = $data['executions']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/functions/' . $functionId . '/usage?range=30d', @@ -1152,7 +1136,6 @@ class UsageTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - sleep(self::WAIT + 20); $tries = 0; while (true) { From 94108e6f3fd9bbc7ba277d6a974e05c36feb2eb9 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:23:34 +0530 Subject: [PATCH 07/11] chore: remove sleep() for usage stats, since batch size is now 1 --- docker-compose.yml | 14 +++++++------- phpunit.xml | 2 +- tests/e2e/General/UsageTest.php | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index facf0e6db9..3316e7dcf8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-assistant: - container_name: appwrite-assistant - image: appwrite/assistant:0.7.0 - networks: - - appwrite - environment: - - _APP_ASSISTANT_OPENAI_API_KEY + # appwrite-assistant: + # container_name: appwrite-assistant + # image: appwrite/assistant:0.7.0 + # networks: + # - appwrite + # environment: + # - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor diff --git a/phpunit.xml b/phpunit.xml index 4c4e55ea4e..598b730908 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 3e22f827bd..8b15461857 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -307,7 +307,7 @@ class UsageTest extends Scope /** * @depends testPrepareStorageStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testStorageStats(array $data): array { $bucketId = $data['bucketId']; @@ -488,6 +488,8 @@ class UsageTest extends Scope ] ); + var_dump($response['body']); + $this->assertEquals($name, $response['body']['name']); $this->assertNotEmpty($response['body']['$id']); From f25df670021e9a5c801ebf4cf4dec871b5a6814f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:39:37 +0530 Subject: [PATCH 08/11] chore: remove sleep() for usage stats, since batch size is now 1 --- tests/e2e/General/UsageTest.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 8b15461857..df780d8f47 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -23,7 +23,7 @@ class UsageTest extends Scope use SideServer; use FunctionsBase; - private const WAIT = 35; + private const WAIT = 5; private const CREATE = 20; protected string $projectId; @@ -470,6 +470,8 @@ class UsageTest extends Scope $this->assertEquals('name', $response['body']['key']); + sleep(self::WAIT); + $requestsTotal += 1; for ($i = 0; $i < self::CREATE; $i++) { @@ -488,8 +490,6 @@ class UsageTest extends Scope ] ); - var_dump($response['body']); - $this->assertEquals($name, $response['body']['name']); $this->assertNotEmpty($response['body']['$id']); @@ -535,6 +535,8 @@ class UsageTest extends Scope $collectionsTotal = $data['collectionsTotal']; $documentsTotal = $data['documentsTotal']; + sleep(self::WAIT); + $response = $this->client->call( Client::METHOD_GET, '/project/usage', From 4c492723892ed80c848a230bd662165d21bf397a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:40:54 +0530 Subject: [PATCH 09/11] chore: enable assistant --- docker-compose.yml | 14 +++++++------- phpunit.xml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3316e7dcf8..facf0e6db9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - # appwrite-assistant: - # container_name: appwrite-assistant - # image: appwrite/assistant:0.7.0 - # networks: - # - appwrite - # environment: - # - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-assistant: + container_name: appwrite-assistant + image: appwrite/assistant:0.7.0 + networks: + - appwrite + environment: + - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor diff --git a/phpunit.xml b/phpunit.xml index 598b730908..4c4e55ea4e 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true" + stopOnFailure="false" > From 3f50532c916c4a4ceeff9935968e88a5ef668b48 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:42:53 +0530 Subject: [PATCH 10/11] chore: remove sleep in functions tests --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 3ed4ca727e..806d07f9f7 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1686,9 +1686,6 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(1, count($executions['body']['executions'])); }); - // Await Aggregation - sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30)); - $this->assertEventually(function () use ($functionId) { $response = $this->getFunctionUsage($functionId, [ 'range' => '24h' From 895807032725091310e8cfdfe5a4fca54e08fa09 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:48:33 +0530 Subject: [PATCH 11/11] chore: linter --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 806d07f9f7..182e6902a3 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -13,7 +13,6 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; -use Utopia\System\System; class FunctionsCustomServerTest extends Scope {