Merge pull request #9397 from appwrite/track-options-requests

Track options requests

app/controllers/general.php

@@ -737,7 +737,8 @@ App::options()
     ->inject('geodb')
     ->inject('isResourceBlocked')
     ->inject('previewHostname')
-    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
+    ->inject('project')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) {
         /*
         * Appwrite Router
         */
@@ -760,6 +761,16 @@ App::options()
             ->addHeader('Access-Control-Allow-Origin', $origin)
             ->addHeader('Access-Control-Allow-Credentials', 'true')
             ->noContent();
+
+        /** OPTIONS requests in utopia do not execute shutdown handlers, as a result we need to track the OPTIONS requests explicitly
+         * @see https://github.com/utopia-php/http/blob/0.33.16/src/App.php#L825-L855
+         */
+        $queueForStatsUsage
+            ->addMetric(METRIC_NETWORK_REQUESTS, 1)
+            ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize())
+            ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize())
+            ->setProject($project)
+            ->trigger();
     });
 
 App::error()
@@ -874,7 +885,10 @@ App::error()
             }
         }
 
-        if ($publish && $project->getId() !== 'console') {
+        /**
+         * If its not a publishable error, track usage stats. Publishable errors are >= 500 or those explicitly marked as publish=true in errors.php
+         */
+        if (!$publish && $project->getId() !== 'console') {
             if (!Auth::isPrivilegedUser(Authorization::getRoles())) {
                 $fileSize = 0;
                 $file = $request->getFiles('file');

src/Appwrite/Platform/Workers/StatsUsage.php

@@ -17,13 +17,20 @@ class StatsUsage extends Action
     private int $lastTriggeredTime = 0;
     private int $keys = 0;
     private const INFINITY_PERIOD = '_inf_';
-    private const KEYS_THRESHOLD = 10000;
+    private const BATCH_SIZE_DEVELOPMENT = 1;
+    private const BATCH_SIZE_PRODUCTION = 10_000;
 
     public static function getName(): string
     {
         return 'stats-usage';
     }
 
+    private function getBatchSize(): int
+    {
+        return System::getEnv('_APP_ENV', 'development') === 'development'
+            ? self::BATCH_SIZE_DEVELOPMENT
+            : self::BATCH_SIZE_PRODUCTION;
+    }
     /**
      * @throws Exception
      */
@@ -86,7 +93,7 @@ class StatsUsage extends Action
 
         // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
         if (
-            $this->keys >= self::KEYS_THRESHOLD ||
+            $this->keys >= $this->getBatchSize() ||
             (time() - $this->lastTriggeredTime > $aggregationInterval  && $this->keys > 0)
         ) {
             Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');

tests/e2e/General/UsageTest.php

@@ -23,7 +23,7 @@ class UsageTest extends Scope
     use SideServer;
     use FunctionsBase;
 
-    private const WAIT = 35;
+    private const WAIT = 5;
     private const CREATE = 20;
 
     protected string $projectId;
@@ -134,8 +134,6 @@ class UsageTest extends Scope
     #[Retry(count: 1)]
     public function testUsersStats(array $data): array
     {
-        sleep(self::WAIT);
-
         $requestsTotal = $data['requestsTotal'];
 
         $response = $this->client->call(
@@ -309,7 +307,7 @@ class UsageTest extends Scope
     /**
      * @depends testPrepareStorageStats
      */
-    #[Retry(count: 1)]
+    #[Retry(count: 10)]
     public function testStorageStats(array $data): array
     {
         $bucketId      = $data['bucketId'];
@@ -318,8 +316,6 @@ class UsageTest extends Scope
         $storageTotal  = $data['storageTotal'];
         $filesTotal    = $data['filesTotal'];
 
-        sleep(self::WAIT);
-
         $response = $this->client->call(
             Client::METHOD_GET,
             '/project/usage',
@@ -474,10 +470,10 @@ class UsageTest extends Scope
 
         $this->assertEquals('name', $response['body']['key']);
 
-        $requestsTotal += 1;
-
         sleep(self::WAIT);
 
+        $requestsTotal += 1;
+
         for ($i = 0; $i < self::CREATE; $i++) {
             $name = uniqid() . ' collection';
 
@@ -709,8 +705,6 @@ class UsageTest extends Scope
     //         $this->assertEquals(201, $response['headers']['status-code']);
     //     }
 
-    //     sleep(self::WAIT);
-
     //     for ($i = 0; $i < 3; $i++) {
     //         try {
     //             $newProjectMetrics = $this->client->call(
@@ -752,7 +746,6 @@ class UsageTest extends Scope
     //             if ($i === 2) {
     //                 throw $e;
     //             }
-    //             sleep(self::WAIT);
     //             continue;
     //         }
     //     }
@@ -792,8 +785,6 @@ class UsageTest extends Scope
     //         $this->assertEquals(204, $response['headers']['status-code']);
     //     }
 
-    //     sleep(self::WAIT);
-
     //     for ($i = 0; $i < 3; $i++) {
     //         try {
     //             $newProjectMetrics = $this->client->call(
@@ -835,7 +826,6 @@ class UsageTest extends Scope
     //             if ($i === 2) {
     //                 throw $e;
     //             }
-    //             sleep(self::WAIT);
     //             continue;
     //         }
     //     }
@@ -1027,8 +1017,6 @@ class UsageTest extends Scope
         $executionTime = $data['executionTime'];
         $executions = $data['executions'];
 
-        sleep(self::WAIT);
-
         $response = $this->client->call(
             Client::METHOD_GET,
             '/functions/' . $functionId . '/usage?range=30d',
@@ -1152,7 +1140,6 @@ class UsageTest extends Scope
 
         $this->assertEquals(200, $response['headers']['status-code']);
 
-        sleep(self::WAIT + 20);
         $tries = 0;
 
         while (true) {

tests/e2e/Services/Functions/FunctionsCustomServerTest.php

@@ -13,7 +13,6 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Role;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Datetime as DatetimeValidator;
-use Utopia\System\System;
 
 class FunctionsCustomServerTest extends Scope
 {
@@ -1686,9 +1685,6 @@ class FunctionsCustomServerTest extends Scope
             $this->assertEquals(1, count($executions['body']['executions']));
         });
 
-        // Await Aggregation
-        sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30));
-
         $this->assertEventually(function () use ($functionId) {
             $response = $this->getFunctionUsage($functionId, [
                 'range' => '24h'