From 5c0579fbc7b0561348f00baee0efa86f2a2c27ee Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 31 Jul 2025 13:50:34 +0530 Subject: [PATCH 1/6] feat: telemetry for cache hits and misses --- app/controllers/shared/api.php | 7 ++++++- app/init/resources.php | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 86fb1e5822..ca7fc4e6ec 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -31,6 +31,7 @@ use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Queue\Publisher; use Utopia\System\System; +use Utopia\Telemetry\Counter; use Utopia\Validator\WhiteList; $parseLabel = function (string $label, array $responsePayload, array $requestParams, Document $user) { @@ -421,7 +422,9 @@ App::init() ->inject('apiKey') ->inject('plan') ->inject('devKey') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('cacheHitsCounter') + ->inject('cacheMissesCounter') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Counter $cacheHitsCounter, Counter $cacheMissesCounter) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -619,10 +622,12 @@ App::init() ->addHeader('Cache-Control', sprintf('private, max-age=%d', $timestamp)) ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); + $cacheHitsCounter->add(1); if (!$isImageTransformation || !$isDisabled) { $response->send($data); } } else { + $cacheMissesCounter->add(1); $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Pragma', 'no-cache') diff --git a/app/init/resources.php b/app/init/resources.php index 162eab1973..5aa8727e44 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1009,3 +1009,19 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef $referrer = (!empty($protocol) ? $protocol : $request->getProtocol()) . '://' . $origin . (!empty($port) ? ':' . $port : ''); return $referrer; }, ['request', 'httpReferrer', 'platforms', 'dbForPlatform', 'project', 'utopia']); + +App::setResource('cacheHitsCounter', function (Telemetry $telemetry) { + return $telemetry->createCounter( + 'cache_hits_total', + 'hits', + 'Total number of cache hits' + ); +}, ['telemetry']); + +App::setResource('cacheMissesCounter', function (Telemetry $telemetry) { + return $telemetry->createCounter( + 'cache_misses_total', + 'misses', + 'Total number of cache misses' + ); +}, ['telemetry']); From d1c3a84ba160df49552c509a60bff98effdb229d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 31 Jul 2025 16:40:40 +0530 Subject: [PATCH 2/6] chore: update to use gauge --- app/controllers/shared/api.php | 14 ++++++++------ app/init/resources.php | 16 ++++------------ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index ca7fc4e6ec..e1641040c8 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -31,7 +31,7 @@ use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Queue\Publisher; use Utopia\System\System; -use Utopia\Telemetry\Counter; +use Utopia\Telemetry\Gauge; use Utopia\Validator\WhiteList; $parseLabel = function (string $label, array $responsePayload, array $requestParams, Document $user) { @@ -422,9 +422,9 @@ App::init() ->inject('apiKey') ->inject('plan') ->inject('devKey') - ->inject('cacheHitsCounter') - ->inject('cacheMissesCounter') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Counter $cacheHitsCounter, Counter $cacheMissesCounter) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('cacheHitsGauge') + ->inject('cacheMissesGauge') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Gauge $cacheHitsCounter, Gauge $cacheMissesCounter) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -622,12 +622,14 @@ App::init() ->addHeader('Cache-Control', sprintf('private, max-age=%d', $timestamp)) ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); - $cacheHitsCounter->add(1); + $cacheHitsCounter->record(1, [ + 'resourceType' => $type, + ]); if (!$isImageTransformation || !$isDisabled) { $response->send($data); } } else { - $cacheMissesCounter->add(1); + $cacheMissesCounter->record(1); $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Pragma', 'no-cache') diff --git a/app/init/resources.php b/app/init/resources.php index 5aa8727e44..c68a8d389d 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1010,18 +1010,10 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef return $referrer; }, ['request', 'httpReferrer', 'platforms', 'dbForPlatform', 'project', 'utopia']); -App::setResource('cacheHitsCounter', function (Telemetry $telemetry) { - return $telemetry->createCounter( - 'cache_hits_total', - 'hits', - 'Total number of cache hits' - ); +App::setResource('cacheHitsGauge', function (Telemetry $telemetry) { + return $telemetry->createGauge('cache.hits.total'); }, ['telemetry']); -App::setResource('cacheMissesCounter', function (Telemetry $telemetry) { - return $telemetry->createCounter( - 'cache_misses_total', - 'misses', - 'Total number of cache misses' - ); +App::setResource('cacheMissesGauge', function (Telemetry $telemetry) { + return $telemetry->createGauge('cache.misses.total'); }, ['telemetry']); From da1fa09e9ce14b0c8568fb2a274b69c7af5031b0 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 31 Jul 2025 17:06:29 +0530 Subject: [PATCH 3/6] chore: update name --- app/controllers/shared/api.php | 10 +++++----- app/init/resources.php | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e1641040c8..a37d921d40 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -422,9 +422,9 @@ App::init() ->inject('apiKey') ->inject('plan') ->inject('devKey') - ->inject('cacheHitsGauge') - ->inject('cacheMissesGauge') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Gauge $cacheHitsCounter, Gauge $cacheMissesCounter) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('storageCacheHitsGauge') + ->inject('storageCacheMissesGauge') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Gauge $storageCacheHitsCounter, Gauge $storageCacheMissesCounter) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -622,14 +622,14 @@ App::init() ->addHeader('Cache-Control', sprintf('private, max-age=%d', $timestamp)) ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); - $cacheHitsCounter->record(1, [ + $storageCacheHitsCounter->record(1, [ 'resourceType' => $type, ]); if (!$isImageTransformation || !$isDisabled) { $response->send($data); } } else { - $cacheMissesCounter->record(1); + $storageCacheMissesCounter->record(1); $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Pragma', 'no-cache') diff --git a/app/init/resources.php b/app/init/resources.php index c68a8d389d..a01fd0e13e 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1010,10 +1010,10 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef return $referrer; }, ['request', 'httpReferrer', 'platforms', 'dbForPlatform', 'project', 'utopia']); -App::setResource('cacheHitsGauge', function (Telemetry $telemetry) { - return $telemetry->createGauge('cache.hits.total'); +App::setResource('storageCacheHitsGauge', function (Telemetry $telemetry) { + return $telemetry->createGauge('storage.cache.hits'); }, ['telemetry']); -App::setResource('cacheMissesGauge', function (Telemetry $telemetry) { - return $telemetry->createGauge('cache.misses.total'); +App::setResource('storageCacheMissesGauge', function (Telemetry $telemetry) { + return $telemetry->createGauge('storage.cache.misses'); }, ['telemetry']); From 588087eda6d0487d995e9b565db1248085997f99 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 31 Jul 2025 17:17:15 +0530 Subject: [PATCH 4/6] chore: add total --- app/init/resources.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index a01fd0e13e..6ab44e255e 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1011,9 +1011,9 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef }, ['request', 'httpReferrer', 'platforms', 'dbForPlatform', 'project', 'utopia']); App::setResource('storageCacheHitsGauge', function (Telemetry $telemetry) { - return $telemetry->createGauge('storage.cache.hits'); + return $telemetry->createGauge('storage.cache.hits.total'); }, ['telemetry']); App::setResource('storageCacheMissesGauge', function (Telemetry $telemetry) { - return $telemetry->createGauge('storage.cache.misses'); + return $telemetry->createGauge('storage.cache.misses.total'); }, ['telemetry']); From 81e656e42c81f18a67ec2ba4d54c6a8a91683dfe Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 31 Jul 2025 18:59:17 +0530 Subject: [PATCH 5/6] chore: update counter --- app/controllers/shared/api.php | 14 ++++++++------ app/init/resources.php | 8 ++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a37d921d40..f627d0ddfe 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -31,7 +31,7 @@ use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Queue\Publisher; use Utopia\System\System; -use Utopia\Telemetry\Gauge; +use Utopia\Telemetry\Counter; use Utopia\Validator\WhiteList; $parseLabel = function (string $label, array $responsePayload, array $requestParams, Document $user) { @@ -422,9 +422,8 @@ App::init() ->inject('apiKey') ->inject('plan') ->inject('devKey') - ->inject('storageCacheHitsGauge') - ->inject('storageCacheMissesGauge') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Gauge $storageCacheHitsCounter, Gauge $storageCacheMissesCounter) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('storageCacheOperationsCounter') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Counter $storageCacheOperationsCounter) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -622,14 +621,17 @@ App::init() ->addHeader('Cache-Control', sprintf('private, max-age=%d', $timestamp)) ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); - $storageCacheHitsCounter->record(1, [ + $storageCacheOperationsCounter->add(1, [ + 'result' => 'hit', 'resourceType' => $type, ]); if (!$isImageTransformation || !$isDisabled) { $response->send($data); } } else { - $storageCacheMissesCounter->record(1); + $storageCacheOperationsCounter->add(1, [ + 'result' => 'miss', + ]); $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Pragma', 'no-cache') diff --git a/app/init/resources.php b/app/init/resources.php index 6ab44e255e..ec62a6c0bb 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1010,10 +1010,6 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef return $referrer; }, ['request', 'httpReferrer', 'platforms', 'dbForPlatform', 'project', 'utopia']); -App::setResource('storageCacheHitsGauge', function (Telemetry $telemetry) { - return $telemetry->createGauge('storage.cache.hits.total'); -}, ['telemetry']); - -App::setResource('storageCacheMissesGauge', function (Telemetry $telemetry) { - return $telemetry->createGauge('storage.cache.misses.total'); +App::setResource('storageCacheOperationsCounter', function (Telemetry $telemetry) { + return $telemetry->createCounter('storage.cache.operations.total'); }, ['telemetry']); From 58d556aab52727650a4949802954771f21e6b0c2 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 1 Aug 2025 15:52:27 +0530 Subject: [PATCH 6/6] chore: update metric naming --- app/controllers/shared/api.php | 16 ++++++---------- app/init/resources.php | 4 ---- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index f627d0ddfe..2ef992764f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -31,7 +31,7 @@ use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Queue\Publisher; use Utopia\System\System; -use Utopia\Telemetry\Counter; +use Utopia\Telemetry\Adapter as Telemetry; use Utopia\Validator\WhiteList; $parseLabel = function (string $label, array $responsePayload, array $requestParams, Document $user) { @@ -422,8 +422,8 @@ App::init() ->inject('apiKey') ->inject('plan') ->inject('devKey') - ->inject('storageCacheOperationsCounter') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Counter $storageCacheOperationsCounter) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('telemetry') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -556,6 +556,7 @@ App::init() )); $useCache = $route->getLabel('cache', false); + $storageCacheOperationsCounter = $telemetry->createCounter('storage.cache.operations.load'); if ($useCache) { $route = $utopia->match($request); $isImageTransformation = $route->getPath() === '/v1/storage/buckets/:bucketId/files/:fileId/preview'; @@ -621,17 +622,12 @@ App::init() ->addHeader('Cache-Control', sprintf('private, max-age=%d', $timestamp)) ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); - $storageCacheOperationsCounter->add(1, [ - 'result' => 'hit', - 'resourceType' => $type, - ]); + $storageCacheOperationsCounter->add(1, ['result' => 'hit']); if (!$isImageTransformation || !$isDisabled) { $response->send($data); } } else { - $storageCacheOperationsCounter->add(1, [ - 'result' => 'miss', - ]); + $storageCacheOperationsCounter->add(1, ['result' => 'miss']); $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Pragma', 'no-cache') diff --git a/app/init/resources.php b/app/init/resources.php index ec62a6c0bb..162eab1973 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1009,7 +1009,3 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef $referrer = (!empty($protocol) ? $protocol : $request->getProtocol()) . '://' . $origin . (!empty($port) ? ':' . $port : ''); return $referrer; }, ['request', 'httpReferrer', 'platforms', 'dbForPlatform', 'project', 'utopia']); - -App::setResource('storageCacheOperationsCounter', function (Telemetry $telemetry) { - return $telemetry->createCounter('storage.cache.operations.total'); -}, ['telemetry']);