diff --git a/Dockerfile b/Dockerfile index 2bb9f80d9e..41592ef9fb 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN composer install --ignore-platform-reqs --optimize-autoloader \ --no-plugins --no-scripts --prefer-dist \ `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi` -FROM appwrite/base:0.9.5 AS final +FROM appwrite/base:0.10.1 AS final LABEL maintainer="team@appwrite.io" diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 1ba3204134..a634618e6e 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2717,7 +2717,7 @@ App::get('/v1/account/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index ffe9142251..013e4639cc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -672,7 +672,7 @@ App::get('/v1/databases/:databaseId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -1068,7 +1068,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -3721,7 +3721,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 05bb461faf..9393f1fbfe 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -996,7 +996,7 @@ App::get('/v1/messaging/providers/:providerId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -2226,7 +2226,7 @@ App::get('/v1/messaging/topics/:topicId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -2638,7 +2638,7 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -3405,7 +3405,7 @@ App::get('/v1/messaging/messages/:messageId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 5fbc1a4e08..b45c9fd3b9 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1364,7 +1364,7 @@ App::get('/v1/teams/:teamId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 0bea1a07f9..4a551b7478 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -864,7 +864,7 @@ App::get('/v1/users/:userId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/general.php b/app/controllers/general.php index 6db4a8b28d..65979d3475 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -737,7 +737,8 @@ App::options() ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->inject('project') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { /* * Appwrite Router */ @@ -760,6 +761,16 @@ App::options() ->addHeader('Access-Control-Allow-Origin', $origin) ->addHeader('Access-Control-Allow-Credentials', 'true') ->noContent(); + + /** OPTIONS requests in utopia do not execute shutdown handlers, as a result we need to track the OPTIONS requests explicitly + * @see https://github.com/utopia-php/http/blob/0.33.16/src/App.php#L825-L855 + */ + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize()) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) + ->setProject($project) + ->trigger(); }); App::error() @@ -874,7 +885,10 @@ App::error() } } - if ($publish && $project->getId() !== 'console') { + /** + * If its not a publishable error, track usage stats. Publishable errors are >= 500 or those explicitly marked as publish=true in errors.php + */ + if (!$publish && $project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; $file = $request->getFiles('file'); diff --git a/composer.json b/composer.json index e096f9e4e7..c9b057b061 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", - "utopia-php/messaging": "0.14.*", + "utopia-php/messaging": "0.16.*", "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.3", diff --git a/composer.lock b/composer.lock index 3d145438e2..e2b25321a2 100644 --- a/composer.lock +++ b/composer.lock @@ -4121,16 +4121,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.14.1", + "version": "0.16.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "4ba356a3aa382802727f7e13e0f0152bcc1fc535" + "reference": "5f3083697102b1821d6624938186761b1e09c54e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/4ba356a3aa382802727f7e13e0f0152bcc1fc535", - "reference": "4ba356a3aa382802727f7e13e0f0152bcc1fc535", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/5f3083697102b1821d6624938186761b1e09c54e", + "reference": "5f3083697102b1821d6624938186761b1e09c54e", "shasum": "" }, "require": { @@ -4166,9 +4166,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.14.1" + "source": "https://github.com/utopia-php/messaging/tree/0.16.0" }, - "time": "2025-01-28T06:14:28+00:00" + "time": "2025-02-18T08:27:00+00:00" }, { "name": "utopia-php/migration", diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index ac3b9ead73..c318f92cf1 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -62,7 +62,7 @@ class StatsResources extends Action Authorization::disable(); Authorization::setDefaultStatus(false); - $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('3 hours')); /** * For each project that were accessed in last 24 hours */ diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index c4d8b0e8d2..a755f723a0 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -17,13 +17,20 @@ class StatsUsage extends Action private int $lastTriggeredTime = 0; private int $keys = 0; private const INFINITY_PERIOD = '_inf_'; - private const KEYS_THRESHOLD = 10000; + private const BATCH_SIZE_DEVELOPMENT = 1; + private const BATCH_SIZE_PRODUCTION = 10_000; public static function getName(): string { return 'stats-usage'; } + private function getBatchSize(): int + { + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; + } /** * @throws Exception */ @@ -86,7 +93,7 @@ class StatsUsage extends Action // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( - $this->keys >= self::KEYS_THRESHOLD || + $this->keys >= $this->getBatchSize() || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c0d0c80eb1..df780d8f47 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -23,7 +23,7 @@ class UsageTest extends Scope use SideServer; use FunctionsBase; - private const WAIT = 35; + private const WAIT = 5; private const CREATE = 20; protected string $projectId; @@ -134,8 +134,6 @@ class UsageTest extends Scope #[Retry(count: 1)] public function testUsersStats(array $data): array { - sleep(self::WAIT); - $requestsTotal = $data['requestsTotal']; $response = $this->client->call( @@ -309,7 +307,7 @@ class UsageTest extends Scope /** * @depends testPrepareStorageStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testStorageStats(array $data): array { $bucketId = $data['bucketId']; @@ -318,8 +316,6 @@ class UsageTest extends Scope $storageTotal = $data['storageTotal']; $filesTotal = $data['filesTotal']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/project/usage', @@ -474,10 +470,10 @@ class UsageTest extends Scope $this->assertEquals('name', $response['body']['key']); - $requestsTotal += 1; - sleep(self::WAIT); + $requestsTotal += 1; + for ($i = 0; $i < self::CREATE; $i++) { $name = uniqid() . ' collection'; @@ -709,8 +705,6 @@ class UsageTest extends Scope // $this->assertEquals(201, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -752,7 +746,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -792,8 +785,6 @@ class UsageTest extends Scope // $this->assertEquals(204, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -835,7 +826,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -1027,8 +1017,6 @@ class UsageTest extends Scope $executionTime = $data['executionTime']; $executions = $data['executions']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/functions/' . $functionId . '/usage?range=30d', @@ -1152,7 +1140,6 @@ class UsageTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - sleep(self::WAIT + 20); $tries = 0; while (true) { diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 3ed4ca727e..182e6902a3 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -13,7 +13,6 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; -use Utopia\System\System; class FunctionsCustomServerTest extends Scope { @@ -1686,9 +1685,6 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(1, count($executions['body']['executions'])); }); - // Await Aggregation - sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30)); - $this->assertEventually(function () use ($functionId) { $response = $this->getFunctionUsage($functionId, [ 'range' => '24h'