diff --git a/app/controllers/general.php b/app/controllers/general.php index 6db4a8b28d..65979d3475 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -737,7 +737,8 @@ App::options() ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->inject('project') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { /* * Appwrite Router */ @@ -760,6 +761,16 @@ App::options() ->addHeader('Access-Control-Allow-Origin', $origin) ->addHeader('Access-Control-Allow-Credentials', 'true') ->noContent(); + + /** OPTIONS requests in utopia do not execute shutdown handlers, as a result we need to track the OPTIONS requests explicitly + * @see https://github.com/utopia-php/http/blob/0.33.16/src/App.php#L825-L855 + */ + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize()) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) + ->setProject($project) + ->trigger(); }); App::error() @@ -874,7 +885,10 @@ App::error() } } - if ($publish && $project->getId() !== 'console') { + /** + * If its not a publishable error, track usage stats. Publishable errors are >= 500 or those explicitly marked as publish=true in errors.php + */ + if (!$publish && $project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; $file = $request->getFiles('file'); diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index c4d8b0e8d2..a755f723a0 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -17,13 +17,20 @@ class StatsUsage extends Action private int $lastTriggeredTime = 0; private int $keys = 0; private const INFINITY_PERIOD = '_inf_'; - private const KEYS_THRESHOLD = 10000; + private const BATCH_SIZE_DEVELOPMENT = 1; + private const BATCH_SIZE_PRODUCTION = 10_000; public static function getName(): string { return 'stats-usage'; } + private function getBatchSize(): int + { + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; + } /** * @throws Exception */ @@ -86,7 +93,7 @@ class StatsUsage extends Action // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( - $this->keys >= self::KEYS_THRESHOLD || + $this->keys >= $this->getBatchSize() || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c0d0c80eb1..df780d8f47 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -23,7 +23,7 @@ class UsageTest extends Scope use SideServer; use FunctionsBase; - private const WAIT = 35; + private const WAIT = 5; private const CREATE = 20; protected string $projectId; @@ -134,8 +134,6 @@ class UsageTest extends Scope #[Retry(count: 1)] public function testUsersStats(array $data): array { - sleep(self::WAIT); - $requestsTotal = $data['requestsTotal']; $response = $this->client->call( @@ -309,7 +307,7 @@ class UsageTest extends Scope /** * @depends testPrepareStorageStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testStorageStats(array $data): array { $bucketId = $data['bucketId']; @@ -318,8 +316,6 @@ class UsageTest extends Scope $storageTotal = $data['storageTotal']; $filesTotal = $data['filesTotal']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/project/usage', @@ -474,10 +470,10 @@ class UsageTest extends Scope $this->assertEquals('name', $response['body']['key']); - $requestsTotal += 1; - sleep(self::WAIT); + $requestsTotal += 1; + for ($i = 0; $i < self::CREATE; $i++) { $name = uniqid() . ' collection'; @@ -709,8 +705,6 @@ class UsageTest extends Scope // $this->assertEquals(201, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -752,7 +746,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -792,8 +785,6 @@ class UsageTest extends Scope // $this->assertEquals(204, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -835,7 +826,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -1027,8 +1017,6 @@ class UsageTest extends Scope $executionTime = $data['executionTime']; $executions = $data['executions']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/functions/' . $functionId . '/usage?range=30d', @@ -1152,7 +1140,6 @@ class UsageTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - sleep(self::WAIT + 20); $tries = 0; while (true) { diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 3ed4ca727e..182e6902a3 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -13,7 +13,6 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; -use Utopia\System\System; class FunctionsCustomServerTest extends Scope { @@ -1686,9 +1685,6 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(1, count($executions['body']['executions'])); }); - // Await Aggregation - sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30)); - $this->assertEventually(function () use ($functionId) { $response = $this->getFunctionUsage($functionId, [ 'range' => '24h'