From 30bc70b9b182f9b11d9885516bdb874d87766e42 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Feb 2025 17:49:22 +1300 Subject: [PATCH 01/21] Abstract key decoding and check if usage is enabled --- app/controllers/api/functions.php | 2 +- app/controllers/shared/api.php | 193 ++++++++----------- composer.json | 2 +- src/Appwrite/Auth/Key.php | 142 ++++++++++++++ src/Appwrite/Platform/Workers/Migrations.php | 21 +- 5 files changed, 225 insertions(+), 135 deletions(-) create mode 100644 src/Appwrite/Auth/Key.php diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 14255ef7a4..cdaf062788 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1277,9 +1277,9 @@ App::post('/v1/functions/:functionId/deployments') model: Response::MODEL_DEPLOYMENT, ) ], - requestType: 'multipart/form-data', type: MethodType::UPLOAD, packaging: true, + requestType: 'multipart/form-data', )) ->param('functionId', '', new UID(), 'Function ID.') ->param('entrypoint', null, new Text(1028), 'Entrypoint File.', true) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index bcee2550c2..3f464ae228 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -3,6 +3,7 @@ use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; +use Appwrite\Auth\Key; use Appwrite\Auth\MFA\Type\TOTP; use Appwrite\Event\Audit; use Appwrite\Event\Build; @@ -15,7 +16,9 @@ use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; +use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; +use Appwrite\SDK\Method; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Abuse\Abuse; @@ -202,117 +205,73 @@ App::init() throw new Exception(Exception::PROJECT_NOT_FOUND); } - /** Default role */ $roles = Config::getParam('roles', []); - $role = ($user->isEmpty()) + + $role = $user->isEmpty() ? Role::guests()->toString() : Role::users()->toString(); - /** Allowed Scopes for the role */ $scopes = $roles[$role]['scopes']; - $apiKey = $request->getHeader('x-appwrite-key', ''); + $apiKey = $request->getHeader('x-appwrite-key'); + + if (!empty($apiKey) && !$user->isEmpty()) { + throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); + } // API Key authentication if (!empty($apiKey)) { - // Do not allow API key and session to be set at the same time - if (!$user->isEmpty()) { - throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); - } + $key = Key::decode($project, $apiKey); - // Remove after migration - if (!\str_contains($apiKey, '_')) { - $keyType = API_KEY_STANDARD; - $authKey = $apiKey; - } else { - [ $keyType, $authKey ] = \explode('_', $apiKey, 2); - } + $scopes = $key->getScopes(); - if ($keyType === API_KEY_DYNAMIC) { - // Dynamic key + $user = new Document([ + '$id' => '', + 'status' => true, + 'type' => Auth::ACTIVITY_TYPE_APP, + 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), + 'password' => '', + 'name' => $key->getName(), + ]); - $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 86400, 0); + // Disable authorization checks for API keys + Authorization::setRole($key->getRole()); + Authorization::setDefaultStatus(false); - try { - $payload = $jwtObj->decode($authKey); - } catch (JWTException $error) { - throw new Exception(Exception::API_KEY_EXPIRED); + if ($key->getType() === API_KEY_STANDARD) { + $dbKey = $project->find( + key: 'secret', + find: $apiKey, + subject: 'keys' + ); + + $accessedAt = $dbKey->getAttribute('accessedAt', ''); + + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { + $dbKey->setAttribute('accessedAt', DateTime::now()); + $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); + $dbForPlatform->purgeCachedDocument('projects', $project->getId()); } - $projectId = $payload['projectId'] ?? ''; - $tokenScopes = $payload['scopes'] ?? []; + $sdkValidator = new WhiteList($servers, true); + $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); - // JWT includes project ID for better security - if ($projectId === $project->getId()) { - $user = new Document([ - '$id' => '', - 'status' => true, - 'type' => Auth::ACTIVITY_TYPE_APP, - 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), - 'password' => '', - 'name' => 'Dynamic Key', - ]); + if ($sdkValidator->isValid($sdk)) { + $sdks = $dbKey->getAttribute('sdks', []); - $role = Auth::USER_ROLE_APPS; - $scopes = \array_merge($roles[$role]['scopes'], $tokenScopes); + if (!in_array($sdk, $sdks)) { + $sdks[] = $sdk; + $dbKey->setAttribute('sdks', $sdks); - Authorization::setRole(Auth::USER_ROLE_APPS); - Authorization::setDefaultStatus(false); // Cancel security segmentation for API keys. - - $queueForAudits->setUser($user); - } - } elseif ($keyType === API_KEY_STANDARD) { - // No underline means no prefix. Backwards compatibility. - // Regular key - - // Check if given key match project API keys - $key = $project->find('secret', $apiKey, 'keys'); - if ($key) { - $user = new Document([ - '$id' => '', - 'status' => true, - 'type' => Auth::ACTIVITY_TYPE_APP, - 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), - 'password' => '', - 'name' => $key->getAttribute('name', 'UNKNOWN'), - ]); - - $role = Auth::USER_ROLE_APPS; - $scopes = \array_merge($roles[$role]['scopes'], $key->getAttribute('scopes', [])); - - $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) { - throw new Exception(Exception::PROJECT_KEY_EXPIRED); - } - - Authorization::setRole(Auth::USER_ROLE_APPS); - Authorization::setDefaultStatus(false); // Cancel security segmentation for API keys. - - $accessedAt = $key->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { - $key->setAttribute('accessedAt', DateTime::now()); - $dbForPlatform->updateDocument('keys', $key->getId(), $key); + /** Update access time as well */ + $dbKey->setAttribute('accessedAt', Datetime::now()); + $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); } - - $sdkValidator = new WhiteList($servers, true); - $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); - if ($sdkValidator->isValid($sdk)) { - $sdks = $key->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { - array_push($sdks, $sdk); - $key->setAttribute('sdks', $sdks); - - /** Update access time as well */ - $key->setAttribute('accessedAt', Datetime::now()); - $dbForPlatform->updateDocument('keys', $key->getId(), $key); - $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - } - } - - $queueForAudits->setUser($user); } } + + $queueForAudits->setUser($user); } // Admin User Authentication elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) { @@ -330,7 +289,7 @@ App::init() throw new Exception(Exception::USER_UNAUTHORIZED); } - $scopes = []; // reset scope if admin + $scopes = []; // Reset scope if admin foreach ($adminRoles as $role) { $scopes = \array_merge($scopes, $roles[$role]['scopes']); } @@ -345,9 +304,7 @@ App::init() Authorization::setRole($authRole); } - /** - * Update project last activity - */ + // Update project last activity if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { @@ -356,9 +313,7 @@ App::init() } } - /** - * Update user last activity - */ + // Update user last activity if (!empty($user->getId())) { $accessedAt = $user->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { @@ -372,18 +327,18 @@ App::init() } } - /** Do not allow access to disabled services */ /** - * @var ?\Appwrite\SDK\Method $method + * @var ?Method $method */ $method = $route->getLabel('sdk', false); - if (is_array($method)) { + if (\is_array($method)) { $method = $method[0]; } if (!empty($method)) { $namespace = $method->getNamespace(); + if ( array_key_exists($namespace, $project->getAttribute('services', [])) && !$project->getAttribute('services', [])[$namespace] @@ -393,13 +348,13 @@ App::init() } } - /** Do now allow access if scope is not allowed */ + // Do now allow access if scope is not allowed $scope = $route->getLabel('scope', 'none'); if (!\in_array($scope, $scopes)) { throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, $user->getAttribute('email', 'User') . ' (role: ' . \strtolower($roles[$role]['label']) . ') missing scope (' . $scope . ')'); } - /** Do not allow access to blocked accounts */ + // Do not allow access to blocked accounts if (false === $user->getAttribute('status')) { // Account is blocked throw new Exception(Exception::USER_BLOCKED); } @@ -471,7 +426,7 @@ App::init() ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); + ->setParam('{chunkId}', (int)($start / ($end + 1 - $start))); $timeLimitArray[] = $timeLimit; } @@ -481,6 +436,13 @@ App::init() $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); + if ($isAppUser) { + $key = Key::decode( + $project, + $request->getHeader('x-appwrite-key') + ); + } + foreach ($timeLimitArray as $timeLimit) { foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys if (!empty($value)) { @@ -549,18 +511,21 @@ App::init() $queueForWebhooks = new Webhook($publisher); $queueForRealtime = new Realtime(); - $dbForProject - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( - $project, - $document, - $response, - $queueForEventsClone->from($queueForEvents), - $queueForFunctions->from($queueForEvents), - $queueForWebhooks->from($queueForEvents), - $queueForRealtime->from($queueForEvents) - )); + if (isset($key) && $key->getUsage()) { + $dbForProject + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)); + } + + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn($event, $document) => $eventDatabaseListener( + $project, + $document, + $response, + $queueForEventsClone->from($queueForEvents), + $queueForFunctions->from($queueForEvents), + $queueForWebhooks->from($queueForEvents), + $queueForRealtime->from($queueForEvents) + )); $useCache = $route->getLabel('cache', false); if ($useCache) { diff --git a/composer.json b/composer.json index fe3400cfe5..f0760354d8 100644 --- a/composer.json +++ b/composer.json @@ -84,7 +84,7 @@ }, "require-dev": { "ext-fileinfo": "*", - "appwrite/sdk-generator": "0.39.32", + "appwrite/sdk-generator": "0.40.*", "phpunit/phpunit": "9.5.20", "swoole/ide-helper": "5.1.2", "textalk/websocket": "1.5.7", diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php new file mode 100644 index 0000000000..f01b26371d --- /dev/null +++ b/src/Appwrite/Auth/Key.php @@ -0,0 +1,142 @@ +projectId; + } + + public function getType(): string + { + return $this->type; + } + + public function getRole(): string + { + return $this->role; + } + + public function getScopes(): array + { + return $this->scopes; + } + + public function getName(): string + { + return $this->name; + } + + public function getUsage(): bool + { + return $this->usage; + } + + public static function decode( + Document $project, + string $key + ): Key + { + if (\str_contains($key, '_')) { + [$type, $secret] = \explode('_', $key, 2); + } else { + $type = API_KEY_STANDARD; + $secret = $key; + } + + $role = Auth::USER_ROLE_APPS; + $roles = Config::getParam('roles', []); + $scopes = $roles[Auth::USER_ROLE_APPS]['scopes'] ?? []; + + $guestKey = new Key( + $project->getId(), + $type, + Auth::USER_ROLE_GUESTS, + $roles[Auth::USER_ROLE_GUESTS]['scopes'] ?? [], + 'UNKNOWN' + ); + + switch ($type) { + case API_KEY_DYNAMIC: + $jwtObj = new JWT( + key: System::getEnv('_APP_OPENSSL_KEY_V1'), + algo: 'HS256', + maxAge: 86400, + leeway: 0 + ); + + try { + $payload = $jwtObj->decode($secret); + } catch (JWTException) { + throw new Exception(Exception::API_KEY_EXPIRED); + } + + $name = $payload['name'] ?? 'Dynamic Key'; + $projectId = $payload['projectId'] ?? ''; + $usage = $payload['usage'] ?? true; + $scopes = \array_merge($payload['scopes'] ?? [], $scopes); + + if ($projectId !== $project->getId()) { + return $guestKey; + } + + return new Key( + $projectId, + $type, + $role, + $scopes, + $name, + $usage + ); + case API_KEY_STANDARD: + $key = $project->find( + key: 'secret', + find: $key, + subject: 'keys' + ); + + if (!$key) { + return $guestKey; + } + + $expire = $key->getAttribute('expire'); + if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) { + throw new Exception(Exception::PROJECT_KEY_EXPIRED); + } + + $name = $key->getAttribute('name', 'UNKNOWN'); + $scopes = \array_merge($key->getAttribute('scopes', []), $scopes); + + return new Key( + $project->getId(), + $type, + $role, + $scopes, + $name + ); + default: + return $guestKey; + } + } +} \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index cd567f6fa3..4cae89f352 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -188,21 +188,6 @@ class Migrations extends Action } /** - * @throws \Utopia\Database\Exception - * @throws Authorization - * @throws Conflict - * @throws Restricted - * @throws Structure - */ - protected function removeAPIKey(Document $apiKey): void - { - $this->dbForPlatform->deleteDocument('keys', $apiKey->getId()); - } - - /** - * @throws Authorization - * @throws Structure - * @throws \Utopia\Database\Exception * @throws Exception */ protected function generateAPIKey(Document $project): string @@ -210,6 +195,7 @@ class Migrations extends Action $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 86400, 0); $apiKey = $jwt->encode([ 'projectId' => $project->getId(), + 'usage' => false, 'scopes' => [ 'users.read', 'users.write', @@ -222,12 +208,9 @@ class Migrations extends Action 'functions.read', 'functions.write', 'databases.read', - 'databases.write', 'collections.read', - 'collections.write', 'documents.read', - 'documents.write' - ] + ], ]); return API_KEY_DYNAMIC . '_' . $apiKey; From e5a3866065689499e7d3d44944cc745778be7fcd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Feb 2025 17:53:40 +1300 Subject: [PATCH 02/21] Fix imports --- app/controllers/shared/api.php | 9 +++------ composer.lock | 14 +++++++------- src/Appwrite/Auth/Key.php | 8 +++----- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 3f464ae228..00a7e2a88d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -1,7 +1,5 @@ getUsage()) { $dbForProject - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)); + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)); } - $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn($event, $document) => $eventDatabaseListener( + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( $project, $document, $response, diff --git a/composer.lock b/composer.lock index fb9dd09359..df834c4c02 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "232691925e05350c7a3831a4e43d79d1", + "content-hash": "53423a0249dc52d6d27bc26dc9425206", "packages": [ { "name": "adhocore/jwt", @@ -5051,16 +5051,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.32", + "version": "0.40.0", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "2d02e1305ea5004fb0aec6b2618d6c597659b75c" + "reference": "d2880132c900f64108d3e4484a6c1ed1bed2303c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/2d02e1305ea5004fb0aec6b2618d6c597659b75c", - "reference": "2d02e1305ea5004fb0aec6b2618d6c597659b75c", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d2880132c900f64108d3e4484a6c1ed1bed2303c", + "reference": "d2880132c900f64108d3e4484a6c1ed1bed2303c", "shasum": "" }, "require": { @@ -5096,9 +5096,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.32" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.0" }, - "time": "2025-01-29T04:04:19+00:00" + "time": "2025-02-04T12:47:33+00:00" }, { "name": "doctrine/annotations", diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index f01b26371d..08918a783c 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -19,8 +19,7 @@ class Key protected array $scopes, protected string $name, protected bool $usage = false, - ) - { + ) { } public function getProjectId(): string @@ -56,8 +55,7 @@ class Key public static function decode( Document $project, string $key - ): Key - { + ): Key { if (\str_contains($key, '_')) { [$type, $secret] = \explode('_', $key, 2); } else { @@ -139,4 +137,4 @@ class Key return $guestKey; } } -} \ No newline at end of file +} From 7490240397b35fb6375173bf72f3563f10085c87 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Feb 2025 18:29:42 +1300 Subject: [PATCH 03/21] Fix name shadowing --- app/controllers/shared/api.php | 4 ++-- src/Appwrite/Auth/Key.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 00a7e2a88d..ee230a83f8 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -434,7 +434,7 @@ App::init() $isAppUser = Auth::isAppUser($roles); if ($isAppUser) { - $key = Key::decode( + $apiKey = Key::decode( $project, $request->getHeader('x-appwrite-key') ); @@ -508,7 +508,7 @@ App::init() $queueForWebhooks = new Webhook($publisher); $queueForRealtime = new Realtime(); - if (isset($key) && $key->getUsage()) { + if (!isset($apiKey) || $apiKey->isUsageEnabled()) { $dbForProject ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)); diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 08918a783c..5fda012eaf 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -47,7 +47,7 @@ class Key return $this->name; } - public function getUsage(): bool + public function isUsageEnabled(): bool { return $this->usage; } From 1c91604c807f84c22ae08efc6dcaa539f1f04d12 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Feb 2025 20:56:39 +1300 Subject: [PATCH 04/21] Only disable database reads/writes and network stats --- app/controllers/api/databases.php | 70 ++++---- app/controllers/api/migrations.php | 6 +- app/controllers/general.php | 15 +- app/controllers/shared/api.php | 95 ++++------ app/init.php | 11 ++ .../Platform/Workers/StatsUsageDump.php | 7 +- tests/e2e/General/UsageTest.php | 7 + .../Services/Migrations/MigrationsBase.php | 164 ++++++++++-------- 8 files changed, 208 insertions(+), 167 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 98ba59106d..f1a3b16d5e 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1,6 +1,7 @@ inject('response') ->inject('dbForProject') ->inject('user') + ->inject('apiKey') ->inject('queueForEvents') ->inject('queueForStatsUsage') - ->inject('mode') - ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, string $mode) { + ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, ?AuthKey $apiKey, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3339,10 +3340,12 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + if (empty($apiKey) || $apiKey->isUsageEnabled()) { + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + } $response->addHeader('X-Debug-Operations', $operations); @@ -3391,9 +3394,9 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->inject('response') ->inject('dbForProject') - ->inject('mode') + ->inject('apiKey') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, ?AuthKey $apiKey, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3505,10 +3508,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + \var_dump('Adding read metrics: ' . $operations); + + if (empty($apiKey) || $apiKey->isUsageEnabled()) { + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); + } $response->addHeader('X-Debug-Operations', $operations); @@ -3570,9 +3576,9 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->inject('response') ->inject('dbForProject') - ->inject('mode') + ->inject('apiKey') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, ?AuthKey $apiKey, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3648,10 +3654,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + if (empty($apiKey) || $apiKey->isUsageEnabled()) { + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); + } $response->addHeader('X-Debug-Operations', $operations); @@ -3803,10 +3810,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') + ->inject('apiKey') ->inject('queueForEvents') - ->inject('mode') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, ?AuthKey $apiKey, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3946,10 +3953,11 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ; + if (empty($apiKey) || $apiKey->isUsageEnabled()) { + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); + } $response->addHeader('X-Debug-Operations', $operations); @@ -4057,10 +4065,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') + ->inject('apiKey') ->inject('queueForEvents') ->inject('queueForStatsUsage') - ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, ?AuthKey $apikey, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -4129,10 +4137,12 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + if (empty($apiKey) || $apiKey->isUsageEnabled()) { + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + } $response->addHeader('X-Debug-Operations', 1); diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index ac149ac8eb..75afc7ed2c 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -48,9 +48,9 @@ App::post('/v1/migrations/appwrite') ] )) ->param('resources', [], new ArrayList(new WhiteList(Appwrite::getSupportedResources())), 'List of resources to migrate') - ->param('endpoint', '', new URL(), "Source's Appwrite Endpoint") - ->param('projectId', '', new UID(), "Source's Project ID") - ->param('apiKey', '', new Text(512), "Source's API Key") + ->param('endpoint', '', new URL(), 'Source Appwrite endpoint') + ->param('projectId', '', new UID(), 'Source Project ID') + ->param('apiKey', '', new Text(512), 'Source API Key') ->inject('response') ->inject('dbForProject') ->inject('project') diff --git a/app/controllers/general.php b/app/controllers/general.php index 1d56e79b74..bb82452609 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -4,6 +4,7 @@ require_once __DIR__ . '/../init.php'; use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; +use Appwrite\Auth\Key; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Event\Func; @@ -770,8 +771,9 @@ App::error() ->inject('project') ->inject('logger') ->inject('log') + ->inject('apiKey') ->inject('queueForStatsUsage') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, StatsUsage $queueForStatsUsage) { + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, ?Key $apiKey, StatsUsage $queueForStatsUsage) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); @@ -882,10 +884,12 @@ App::error() $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); + if (empty($apiKey) || $apiKey->isUsageEnabled()) { + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); + } } $queueForStatsUsage @@ -893,7 +897,6 @@ App::error() ->trigger(); } - if ($logger && $publish) { try { /** @var Utopia\Database\Document $user */ diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index ee230a83f8..8a75773b8d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -98,28 +98,22 @@ $usageDatabaseListener = function (string $event, Document $document, StatsUsage switch (true) { case $document->getCollection() === 'teams': - $queueForStatsUsage - ->addMetric(METRIC_TEAMS, $value); // per project + $queueForStatsUsage->addMetric(METRIC_TEAMS, $value); // per project break; case $document->getCollection() === 'users': - $queueForStatsUsage - ->addMetric(METRIC_USERS, $value); // per project + $queueForStatsUsage->addMetric(METRIC_USERS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage - ->addReduce($document); + $queueForStatsUsage->addReduce($document); } break; case $document->getCollection() === 'sessions': // sessions - $queueForStatsUsage - ->addMetric(METRIC_SESSIONS, $value); //per project + $queueForStatsUsage->addMetric(METRIC_SESSIONS, $value); //per project break; case $document->getCollection() === 'databases': // databases - $queueForStatsUsage - ->addMetric(METRIC_DATABASES, $value); // per project + $queueForStatsUsage->addMetric(METRIC_DATABASES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage - ->addReduce($document); + $queueForStatsUsage->addReduce($document); } break; case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections @@ -131,8 +125,7 @@ $usageDatabaseListener = function (string $event, Document $document, StatsUsage ; if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage - ->addReduce($document); + $queueForStatsUsage->addReduce($document); } break; case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents @@ -195,7 +188,8 @@ App::init() ->inject('servers') ->inject('mode') ->inject('team') - ->action(function (App $utopia, Request $request, Database $dbForPlatform, Database $dbForProject, Audit $queueForAudits, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team) { + ->inject('apiKey') + ->action(function (App $utopia, Request $request, Database $dbForPlatform, Database $dbForProject, Audit $queueForAudits, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey) { $route = $utopia->getRoute(); if ($project->isEmpty()) { @@ -210,17 +204,13 @@ App::init() $scopes = $roles[$role]['scopes']; - $apiKey = $request->getHeader('x-appwrite-key'); - if (!empty($apiKey) && !$user->isEmpty()) { throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); } // API Key authentication if (!empty($apiKey)) { - $key = Key::decode($project, $apiKey); - - $scopes = $key->getScopes(); + $scopes = $apiKey->getScopes(); $user = new Document([ '$id' => '', @@ -228,17 +218,17 @@ App::init() 'type' => Auth::ACTIVITY_TYPE_APP, 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), 'password' => '', - 'name' => $key->getName(), + 'name' => $apiKey->getName(), ]); // Disable authorization checks for API keys - Authorization::setRole($key->getRole()); + Authorization::setRole($apiKey->getRole()); Authorization::setDefaultStatus(false); - if ($key->getType() === API_KEY_STANDARD) { + if ($apiKey->getType() === API_KEY_STANDARD) { $dbKey = $project->find( key: 'secret', - find: $apiKey, + find: $request->getHeader('x-appwrite-key', ''), subject: 'keys' ); @@ -433,13 +423,6 @@ App::init() $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); - if ($isAppUser) { - $apiKey = Key::decode( - $project, - $request->getHeader('x-appwrite-key') - ); - } - foreach ($timeLimitArray as $timeLimit) { foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys if (!empty($value)) { @@ -508,21 +491,18 @@ App::init() $queueForWebhooks = new Webhook($publisher); $queueForRealtime = new Realtime(); - if (!isset($apiKey) || $apiKey->isUsageEnabled()) { - $dbForProject - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)); - } - - $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( - $project, - $document, - $response, - $queueForEventsClone->from($queueForEvents), - $queueForFunctions->from($queueForEvents), - $queueForWebhooks->from($queueForEvents), - $queueForRealtime->from($queueForEvents) - )); + $dbForProject + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) + ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( + $project, + $document, + $response, + $queueForEventsClone->from($queueForEvents), + $queueForFunctions->from($queueForEvents), + $queueForWebhooks->from($queueForEvents), + $queueForRealtime->from($queueForEvents) + )); $useCache = $route->getLabel('cache', false); if ($useCache) { @@ -542,10 +522,7 @@ App::init() $bucketId = $parts[1] ?? null; $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); - $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAppUser && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } @@ -654,6 +631,7 @@ App::shutdown() ->inject('response') ->inject('project') ->inject('user') + ->inject('apiKey') ->inject('queueForEvents') ->inject('queueForAudits') ->inject('queueForStatsUsage') @@ -665,7 +643,7 @@ App::shutdown() ->inject('queueForWebhooks') ->inject('queueForRealtime') ->inject('dbForProject') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, ?Key $apiKey, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -749,6 +727,7 @@ App::shutdown() foreach ($queueForEvents->getParams() as $key => $value) { $queueForAudits->setParam($key, $value); } + $queueForAudits->trigger(); } @@ -768,9 +747,7 @@ App::shutdown() $queueForMessaging->trigger(); } - /** - * Cache label - */ + // Cache label $useCache = $route->getLabel('cache', false); if ($useCache) { $resource = $resourceType = null; @@ -822,10 +799,12 @@ App::shutdown() $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); + if (empty($apiKey) || $apiKey->isUsageEnabled()) { + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); + } } $queueForStatsUsage diff --git a/app/init.php b/app/init.php index 0c6746de3c..354078204a 100644 --- a/app/init.php +++ b/app/init.php @@ -21,6 +21,7 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; +use Appwrite\Auth\Key; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1942,3 +1943,13 @@ App::setResource('previewHostname', function (Request $request) { return ''; }, ['request']); + +App::setResource('apiKey', function(Request $request, Document $project): ?Key { + $key = $request->getHeader('x-appwrite-key'); + + if (empty($key)) { + return null; + } + + return Key::decode($project, $key); +}); diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 38ebd578a5..5d7240f4b5 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -98,8 +98,10 @@ class StatsUsageDump extends Action * @param Message $message * @param callable $getProjectDB * @param callable $getLogsDB + * @param Registry $register * @return void * @throws Exception + * @throws \Throwable * @throws \Utopia\Database\Exception */ public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void @@ -111,7 +113,6 @@ class StatsUsageDump extends Action throw new Exception('Missing payload'); } - foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); @@ -152,7 +153,9 @@ class StatsUsageDump extends Action 'value' => $value, 'region' => System::getEnv('_APP_REGION', 'default'), ]); + $documentClone = new Document($document->getArrayCopy()); + $dbForProject->createOrUpdateDocumentsWithIncrease( 'stats', 'value', @@ -315,7 +318,7 @@ class StatsUsageDump extends Action console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); } - protected function writeToLogsDB(Document $project, Document $document) + protected function writeToLogsDB(Document $project, Document $document): void { if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) { Console::log('Dual Writing is disabled. Skipping...'); diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 74ae1c00bc..c0d0c80eb1 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -52,6 +52,13 @@ class UsageTest extends Scope } } + public static function getYesterday(): string + { + $date = new DateTime(); + $date->modify('-1 day'); + return $date->format(self::$formatTz); + } + public static function getToday(): string { $date = new DateTime(); diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index e4c7ba7712..0875fa236d 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -4,6 +4,7 @@ namespace Tests\E2E\Services\Migrations; use CURLFile; use Tests\E2E\Client; +use Tests\E2E\General\UsageTest; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Database\Helpers\ID; @@ -20,13 +21,13 @@ trait MigrationsBase /** * @var array */ - protected static $destinationProject = []; + protected static array $destinationProject = []; /** * @param bool $fresh * @return array */ - public function getDesintationProject(bool $fresh = false): array + public function getDestinationProject(bool $fresh = false): array { if (!empty(self::$destinationProject) && !$fresh) { return self::$destinationProject; @@ -40,13 +41,11 @@ trait MigrationsBase return self::$destinationProject; } - public function performMigrationSync( - array $body, - ): array { + public function performMigrationSync(array $body): array { $migration = $this->client->call(Client::METHOD_POST, '/migrations/appwrite', [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ], $body); $this->assertEquals(202, $migration['headers']['status-code']); @@ -57,8 +56,8 @@ trait MigrationsBase while ($attempts < 5) { $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migration['body']['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -82,12 +81,14 @@ trait MigrationsBase $attempts++; sleep(5); } + + return []; } /** * Appwrite E2E Migration Tests */ - public function testCreateAppwriteMigration() + public function testCreateAppwriteMigration(): void { $response = $this->performMigrationSync([ 'resources' => Appwrite::getSupportedResources(), @@ -105,7 +106,7 @@ trait MigrationsBase /** * Auth */ - public function testAppwriteMigrationAuthUserPassword() + public function testAppwriteMigrationAuthUserPassword(): void { $response = $this->client->call(Client::METHOD_POST, '/users', [ 'content-type' => 'application/json', @@ -144,8 +145,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/users/' . $user['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -157,8 +158,8 @@ trait MigrationsBase // Cleanup $this->client->call(Client::METHOD_DELETE, '/users/' . $user['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->client->call(Client::METHOD_DELETE, '/users/' . $user['$id'], [ @@ -168,7 +169,7 @@ trait MigrationsBase ]); } - public function testAppwriteMigrationAuthUserPhone() + public function testAppwriteMigrationAuthUserPhone(): void { $response = $this->client->call(Client::METHOD_POST, '/users', [ 'content-type' => 'application/json', @@ -206,8 +207,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/users/' . $user['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -224,12 +225,12 @@ trait MigrationsBase $this->client->call(Client::METHOD_DELETE, '/users/' . $user['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } - public function testAppwriteMigrationAuthTeam() + public function testAppwriteMigrationAuthTeam(): void { $user = $this->client->call(Client::METHOD_POST, '/users', [ 'content-type' => 'application/json', @@ -309,8 +310,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/teams/' . $team['body']['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -320,8 +321,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/teams/' . $team['body']['$id'] . '/memberships', [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -342,8 +343,8 @@ trait MigrationsBase $this->client->call(Client::METHOD_DELETE, '/teams/' . $team['body']['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->client->call(Client::METHOD_DELETE, '/users/' . $user['body']['$id'], [ @@ -354,8 +355,8 @@ trait MigrationsBase $this->client->call(Client::METHOD_DELETE, '/users/' . $user['body']['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->client->call(Client::METHOD_DELETE, '/teams/' . $team['body']['$id'], [ @@ -366,15 +367,15 @@ trait MigrationsBase $this->client->call(Client::METHOD_DELETE, '/teams/' . $team['body']['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } /** * Databases */ - public function testAppwriteMigrationDatabase() + public function testAppwriteMigrationDatabase(): array { $response = $this->client->call(Client::METHOD_POST, '/databases', [ 'content-type' => 'application/json', @@ -400,7 +401,6 @@ trait MigrationsBase 'apiKey' => $this->getProject()['apiKey'], ]); - $this->assertEquals('completed', $result['status']); $this->assertEquals([Resource::TYPE_DATABASE], $result['resources']); $this->assertArrayHasKey(Resource::TYPE_DATABASE, $result['statusCounters']); @@ -412,8 +412,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -426,8 +426,8 @@ trait MigrationsBase // Cleanup on destination $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); return [ @@ -438,7 +438,7 @@ trait MigrationsBase /** * @depends testAppwriteMigrationDatabase */ - public function testAppwriteMigrationDatabasesCollection(array $data) + public function testAppwriteMigrationDatabasesCollection(array $data): array { $databaseId = $data['databaseId']; @@ -506,8 +506,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -518,8 +518,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -532,8 +532,8 @@ trait MigrationsBase // Cleanup $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); return [ @@ -545,7 +545,7 @@ trait MigrationsBase /** * @depends testAppwriteMigrationDatabasesCollection */ - public function testAppwriteMigrationDatabasesDocument(array $data) + public function testAppwriteMigrationDatabasesDocument(array $data): void { $databaseId = $data['databaseId']; $collectionId = $data['collectionId']; @@ -567,6 +567,15 @@ trait MigrationsBase $documentId = $document['body']['$id']; + // Get project stats + $initialStats = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'startDate' => UsageTest::getYesterday(), + 'endDate' => UsageTest::getTomorrow(), + ]); + $result = $this->performMigrationSync([ 'resources' => [ Resource::TYPE_DATABASE, @@ -579,6 +588,25 @@ trait MigrationsBase 'apiKey' => $this->getProject()['apiKey'], ]); + // Wait for usage dump + sleep(36); + + $finalStats = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'startDate' => UsageTest::getYesterday(), + 'endDate' => UsageTest::getTomorrow(), + ]); + + \var_dump($initialStats); + + // Compare database reads/writes + $this->assertGreaterThan(0, $initialStats['body']['databasesReadsTotal']); + $this->assertGreaterThan(0, $initialStats['body']['databasesWritesTotal']); + $this->assertEquals($initialStats['body']['databasesReadsTotal'], $finalStats['body']['databasesReadsTotal']); + $this->assertEquals($initialStats['body']['databasesWritesTotal'], $finalStats['body']['databasesWritesTotal']); + $this->assertEquals('completed', $result['status']); $this->assertEquals([Resource::TYPE_DATABASE, Resource::TYPE_COLLECTION, Resource::TYPE_ATTRIBUTE, Resource::TYPE_DOCUMENT], $result['resources']); @@ -594,8 +622,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -607,15 +635,15 @@ trait MigrationsBase // Cleanup $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } /** * Storage */ - public function testAppwriteMigrationStorageBucket() + public function testAppwriteMigrationStorageBucket(): void { $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ 'content-type' => 'application/json', @@ -663,8 +691,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucket['body']['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -683,8 +711,8 @@ trait MigrationsBase // Cleanup $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucket['body']['$id'], [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucket['body']['$id'], [ @@ -694,7 +722,7 @@ trait MigrationsBase ]); } - public function testAppwriteMigrationStorageFiles() + public function testAppwriteMigrationStorageFiles(): void { $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ 'content-type' => 'application/json', @@ -767,8 +795,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -786,15 +814,15 @@ trait MigrationsBase $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } /** * Functions */ - public function testAppwriteMigrationFunction() + public function testAppwriteMigrationFunction(): void { $functionId = $this->setupFunction([ 'functionId' => ID::unique(), @@ -839,8 +867,8 @@ trait MigrationsBase $response = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -856,8 +884,8 @@ trait MigrationsBase $this->assertEventually(function () use ($functionId) { $deployments = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/', array_merge([ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ])); $this->assertEquals(200, $deployments['headers']['status-code']); @@ -870,8 +898,8 @@ trait MigrationsBase // Attempt execution $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ], [ 'body' => 'test' ]); @@ -888,8 +916,8 @@ trait MigrationsBase $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getDesintationProject()['$id'], - 'x-appwrite-key' => $this->getDesintationProject()['apiKey'], + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } } From 0634a7eac3cdb13935aae2c22c2744a5a8c34642 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Feb 2025 21:13:01 +1300 Subject: [PATCH 05/21] Fix injections --- app/controllers/api/databases.php | 2 -- app/init.php | 2 +- tests/e2e/Services/Migrations/MigrationsBase.php | 8 ++------ 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index f1a3b16d5e..6c2967b590 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3508,8 +3508,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } - \var_dump('Adding read metrics: ' . $operations); - if (empty($apiKey) || $apiKey->isUsageEnabled()) { $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) diff --git a/app/init.php b/app/init.php index 354078204a..faf3a8f70b 100644 --- a/app/init.php +++ b/app/init.php @@ -1952,4 +1952,4 @@ App::setResource('apiKey', function(Request $request, Document $project): ?Key { } return Key::decode($project, $key); -}); +}, ['request', 'project']); diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 0875fa236d..ef150d84dc 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -589,7 +589,7 @@ trait MigrationsBase ]); // Wait for usage dump - sleep(36); + sleep(30); $finalStats = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([ 'content-type' => 'application/json', @@ -599,11 +599,7 @@ trait MigrationsBase 'endDate' => UsageTest::getTomorrow(), ]); - \var_dump($initialStats); - - // Compare database reads/writes - $this->assertGreaterThan(0, $initialStats['body']['databasesReadsTotal']); - $this->assertGreaterThan(0, $initialStats['body']['databasesWritesTotal']); + // Ensure database reads/writes did not change $this->assertEquals($initialStats['body']['databasesReadsTotal'], $finalStats['body']['databasesReadsTotal']); $this->assertEquals($initialStats['body']['databasesWritesTotal'], $finalStats['body']['databasesWritesTotal']); From bed7ab321eb20df7b6b4a95da0d05ab8543df95a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Feb 2025 22:02:20 +1300 Subject: [PATCH 06/21] Add unit test --- src/Appwrite/Auth/Key.php | 9 ++++ src/Appwrite/Platform/Workers/Migrations.php | 1 + tests/unit/Auth/KeyTest.php | 57 ++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 tests/unit/Auth/KeyTest.php diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 5fda012eaf..5143b243bf 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -52,6 +52,15 @@ class Key return $this->usage; } + /** + * Decode the given secret key into a Key object, containing the project ID, type, role, scopes, and name. + * Can be a stored API key or a dynamic key (JWT). + * + * @param Document $project + * @param string $key + * @return Key + * @throws Exception + */ public static function decode( Document $project, string $key diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 4cae89f352..3c0055e637 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -193,6 +193,7 @@ class Migrations extends Action protected function generateAPIKey(Document $project): string { $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 86400, 0); + $apiKey = $jwt->encode([ 'projectId' => $project->getId(), 'usage' => false, diff --git a/tests/unit/Auth/KeyTest.php b/tests/unit/Auth/KeyTest.php new file mode 100644 index 0000000000..78c4866a41 --- /dev/null +++ b/tests/unit/Auth/KeyTest.php @@ -0,0 +1,57 @@ + $projectId,]); + $decoded = Key::decode($project, $key); + + $this->assertEquals($projectId, $decoded->getProjectId()); + $this->assertEquals(API_KEY_DYNAMIC, $decoded->getType()); + $this->assertEquals(Auth::USER_ROLE_APPS, $decoded->getRole()); + $this->assertEquals(\array_merge($scopes, $roleScopes), $decoded->getScopes()); + } + + private static function generateKey( + string $projectId, + bool $usage, + array $scopes, + ): string + { + $jwt = new JWT( + key: System::getEnv('_APP_OPENSSL_KEY_V1'), + algo: 'HS256', + maxAge: 86400, + leeway: 0, + ); + + $apiKey = $jwt->encode([ + 'projectId' => $projectId, + 'usage' => $usage, + 'scopes' => $scopes, + ]); + + return API_KEY_DYNAMIC . '_' . $apiKey; + } +} \ No newline at end of file From 704437b5e8c27c195eb574b24846cdd2b0c239af Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Feb 2025 22:04:52 +1300 Subject: [PATCH 07/21] Format --- app/init.php | 2 +- composer.lock | 4 ++-- tests/e2e/Services/Migrations/MigrationsBase.php | 3 ++- tests/unit/Auth/KeyTest.php | 5 ++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/init.php b/app/init.php index faf3a8f70b..25fa9faa85 100644 --- a/app/init.php +++ b/app/init.php @@ -1944,7 +1944,7 @@ App::setResource('previewHostname', function (Request $request) { return ''; }, ['request']); -App::setResource('apiKey', function(Request $request, Document $project): ?Key { +App::setResource('apiKey', function (Request $request, Document $project): ?Key { $key = $request->getHeader('x-appwrite-key'); if (empty($key)) { diff --git a/composer.lock b/composer.lock index d3971da521..d6253eed5d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "53423a0249dc52d6d27bc26dc9425206", + "content-hash": "a7081f31ed2111c7b3a5a9c25b69bf80", "packages": [ { "name": "adhocore/jwt", @@ -8747,7 +8747,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index ef150d84dc..674f607c0a 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -41,7 +41,8 @@ trait MigrationsBase return self::$destinationProject; } - public function performMigrationSync(array $body): array { + public function performMigrationSync(array $body): array + { $migration = $this->client->call(Client::METHOD_POST, '/migrations/appwrite', [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getDestinationProject()['$id'], diff --git a/tests/unit/Auth/KeyTest.php b/tests/unit/Auth/KeyTest.php index 78c4866a41..8ae2114697 100644 --- a/tests/unit/Auth/KeyTest.php +++ b/tests/unit/Auth/KeyTest.php @@ -37,8 +37,7 @@ class KeyTest extends TestCase string $projectId, bool $usage, array $scopes, - ): string - { + ): string { $jwt = new JWT( key: System::getEnv('_APP_OPENSSL_KEY_V1'), algo: 'HS256', @@ -54,4 +53,4 @@ class KeyTest extends TestCase return API_KEY_DYNAMIC . '_' . $apiKey; } -} \ No newline at end of file +} From d4c659f88da9075931b94cd9aa8022f649de2462 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Feb 2025 18:17:47 +1300 Subject: [PATCH 08/21] Fix key usage default --- src/Appwrite/Auth/Key.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 5143b243bf..7cd0ac91b6 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -18,7 +18,7 @@ class Key protected string $role, protected array $scopes, protected string $name, - protected bool $usage = false, + protected bool $usage = true, ) { } From 5780ec9547668adc96022438e0749ad1a75e72f3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Feb 2025 18:45:06 +1300 Subject: [PATCH 09/21] Make sure key has app role before continuing checks --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8a75773b8d..466cd3987e 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -209,7 +209,7 @@ App::init() } // API Key authentication - if (!empty($apiKey)) { + if (!empty($apiKey) && $apiKey->getRole() === Auth::USER_ROLE_APPS) { $scopes = $apiKey->getScopes(); $user = new Document([ From 775e6ec9cfd877066bc61c6e8ca32b888b0eff05 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Feb 2025 20:34:38 +1300 Subject: [PATCH 10/21] Fix role setting --- app/controllers/shared/api.php | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 466cd3987e..358c96d10d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -121,8 +121,7 @@ $usageDatabaseListener = function (string $event, Document $document, StatsUsage $databaseInternalId = $parts[1] ?? 0; $queueForStatsUsage ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) - ; + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value); if ($event === Database::EVENT_DOCUMENT_DELETE) { $queueForStatsUsage->addReduce($document); @@ -209,9 +208,14 @@ App::init() } // API Key authentication - if (!empty($apiKey) && $apiKey->getRole() === Auth::USER_ROLE_APPS) { + if (!empty($apiKey)) { + $role = $apiKey->getRole(); $scopes = $apiKey->getScopes(); + // Disable authorization checks for API keys + Authorization::setDefaultStatus(false); + + if ($apiKey->getRole() === Auth::USER_ROLE_APPS) { $user = new Document([ '$id' => '', 'status' => true, @@ -221,9 +225,8 @@ App::init() 'name' => $apiKey->getName(), ]); - // Disable authorization checks for API keys - Authorization::setRole($apiKey->getRole()); - Authorization::setDefaultStatus(false); + $queueForAudits->setUser($user); + } if ($apiKey->getType() === API_KEY_STANDARD) { $dbKey = $project->find( @@ -232,6 +235,7 @@ App::init() subject: 'keys' ); + if ($dbKey) { $accessedAt = $dbKey->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { @@ -256,11 +260,11 @@ App::init() $dbForPlatform->purgeCachedDocument('projects', $project->getId()); } } - } $queueForAudits->setUser($user); } - // Admin User Authentication + } + } // Admin User Authentication elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) { $teamId = $team->getId(); $adminRoles = []; @@ -564,8 +568,7 @@ App::init() ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Pragma', 'no-cache') ->addHeader('Expires', '0') - ->addHeader('X-Appwrite-Cache', 'miss') - ; + ->addHeader('X-Appwrite-Cache', 'miss'); } } }); From 9427b07580eff0dc78cc43ab392be815952eafda Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Feb 2025 21:22:08 +1300 Subject: [PATCH 11/21] Add expired property directly to key --- app/controllers/shared/api.php | 11 +++++++---- src/Appwrite/Auth/Key.php | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 358c96d10d..c282419ec9 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -203,12 +203,15 @@ App::init() $scopes = $roles[$role]['scopes']; - if (!empty($apiKey) && !$user->isEmpty()) { - throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); - } - // API Key authentication if (!empty($apiKey)) { + if (!$user->isEmpty()) { + throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); + } + if ($apiKey->isExpired()) { + throw new Exception(Exception::PROJECT_KEY_EXPIRED); + } + $role = $apiKey->getRole(); $scopes = $apiKey->getScopes(); diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 7cd0ac91b6..2eb5b07edc 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -18,6 +18,7 @@ class Key protected string $role, protected array $scopes, protected string $name, + protected bool $expired = false, protected bool $usage = true, ) { } @@ -47,6 +48,11 @@ class Key return $this->name; } + public function isExpired(): bool + { + return $this->expired; + } + public function isUsageEnabled(): bool { return $this->usage; @@ -75,6 +81,7 @@ class Key $role = Auth::USER_ROLE_APPS; $roles = Config::getParam('roles', []); $scopes = $roles[Auth::USER_ROLE_APPS]['scopes'] ?? []; + $expired = false; $guestKey = new Key( $project->getId(), @@ -96,7 +103,7 @@ class Key try { $payload = $jwtObj->decode($secret); } catch (JWTException) { - throw new Exception(Exception::API_KEY_EXPIRED); + $expired = true; } $name = $payload['name'] ?? 'Dynamic Key'; @@ -114,6 +121,7 @@ class Key $role, $scopes, $name, + $expired, $usage ); case API_KEY_STANDARD: @@ -129,7 +137,7 @@ class Key $expire = $key->getAttribute('expire'); if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) { - throw new Exception(Exception::PROJECT_KEY_EXPIRED); + $expired = true; } $name = $key->getAttribute('name', 'UNKNOWN'); @@ -140,7 +148,8 @@ class Key $type, $role, $scopes, - $name + $name, + $expired ); default: return $guestKey; From 2449b76676c710723ab5d3899716c82f0628ac16 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Feb 2025 21:25:53 +1300 Subject: [PATCH 12/21] Format --- app/controllers/shared/api.php | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c282419ec9..24912ad985 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -206,8 +206,8 @@ App::init() // API Key authentication if (!empty($apiKey)) { if (!$user->isEmpty()) { - throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); - } + throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); + } if ($apiKey->isExpired()) { throw new Exception(Exception::PROJECT_KEY_EXPIRED); } @@ -219,14 +219,14 @@ App::init() Authorization::setDefaultStatus(false); if ($apiKey->getRole() === Auth::USER_ROLE_APPS) { - $user = new Document([ - '$id' => '', - 'status' => true, - 'type' => Auth::ACTIVITY_TYPE_APP, - 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), - 'password' => '', - 'name' => $apiKey->getName(), - ]); + $user = new Document([ + '$id' => '', + 'status' => true, + 'type' => Auth::ACTIVITY_TYPE_APP, + 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), + 'password' => '', + 'name' => $apiKey->getName(), + ]); $queueForAudits->setUser($user); } @@ -239,33 +239,33 @@ App::init() ); if ($dbKey) { - $accessedAt = $dbKey->getAttribute('accessedAt', ''); + $accessedAt = $dbKey->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { - $dbKey->setAttribute('accessedAt', DateTime::now()); - $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); - $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - } - - $sdkValidator = new WhiteList($servers, true); - $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); - - if ($sdkValidator->isValid($sdk)) { - $sdks = $dbKey->getAttribute('sdks', []); - - if (!in_array($sdk, $sdks)) { - $sdks[] = $sdk; - $dbKey->setAttribute('sdks', $sdks); - - /** Update access time as well */ - $dbKey->setAttribute('accessedAt', Datetime::now()); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { + $dbKey->setAttribute('accessedAt', DateTime::now()); $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); } - } - $queueForAudits->setUser($user); - } + $sdkValidator = new WhiteList($servers, true); + $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); + + if ($sdkValidator->isValid($sdk)) { + $sdks = $dbKey->getAttribute('sdks', []); + + if (!in_array($sdk, $sdks)) { + $sdks[] = $sdk; + $dbKey->setAttribute('sdks', $sdks); + + /** Update access time as well */ + $dbKey->setAttribute('accessedAt', Datetime::now()); + $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); + $dbForPlatform->purgeCachedDocument('projects', $project->getId()); + } + } + + $queueForAudits->setUser($user); + } } } // Admin User Authentication elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) { From 6dbc8c5464a2728b91335760d0b9c7ba904e0878 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Feb 2025 00:19:51 +1300 Subject: [PATCH 13/21] Allow disabling specific metrics in hooks via keys --- app/controllers/api/databases.php | 44 ++++++++------------ app/controllers/general.php | 10 ++--- app/controllers/shared/api.php | 19 +++++---- src/Appwrite/Auth/Key.php | 10 ++--- src/Appwrite/Event/StatsUsage.php | 18 ++++++++ src/Appwrite/Platform/Workers/Migrations.php | 8 +++- 6 files changed, 63 insertions(+), 46 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index cc84e876d0..80d55aaabf 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3343,12 +3343,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); - if (empty($apiKey) || $apiKey->isUsageEnabled()) { - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - } + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->addHeader('X-Debug-Operations', $operations); @@ -3511,11 +3509,9 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } - if (empty($apiKey) || $apiKey->isUsageEnabled()) { - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); - } + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3655,11 +3651,9 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); - if (empty($apiKey) || $apiKey->isUsageEnabled()) { - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); - } + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3954,11 +3948,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); - if (empty($apiKey) || $apiKey->isUsageEnabled()) { - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); - } + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -4138,12 +4130,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); - if (empty($apiKey) || $apiKey->isUsageEnabled()) { - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - } + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->addHeader('X-Debug-Operations', 1); diff --git a/app/controllers/general.php b/app/controllers/general.php index bb82452609..6da3354400 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -884,12 +884,10 @@ App::error() $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - if (empty($apiKey) || $apiKey->isUsageEnabled()) { - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); - } + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); } $queueForStatsUsage diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 24912ad985..44d66a8c02 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -389,7 +389,8 @@ App::init() ->inject('dbForProject') ->inject('timelimit') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('apiKey') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode, ?Key $apiKey) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -486,6 +487,12 @@ App::init() $queueForAudits->setUser($userClone); } + if (!empty($apiKey) && !empty($apiKey->getDisabledMetrics())) { + foreach ($apiKey->getDisabledMetrics() as $key) { + $queueForStatsUsage->disableMetric($key); + } + } + $queueForDeletes->setProject($project); $queueForDatabase->setProject($project); $queueForBuilds->setProject($project); @@ -805,12 +812,10 @@ App::shutdown() $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - if (empty($apiKey) || $apiKey->isUsageEnabled()) { - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); - } + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); } $queueForStatsUsage diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 2eb5b07edc..1c40b35f54 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -19,7 +19,7 @@ class Key protected array $scopes, protected string $name, protected bool $expired = false, - protected bool $usage = true, + protected array $disabledMetrics = [], ) { } @@ -53,9 +53,9 @@ class Key return $this->expired; } - public function isUsageEnabled(): bool + public function getDisabledMetrics(): array { - return $this->usage; + return $this->disabledMetrics; } /** @@ -108,7 +108,7 @@ class Key $name = $payload['name'] ?? 'Dynamic Key'; $projectId = $payload['projectId'] ?? ''; - $usage = $payload['usage'] ?? true; + $disabledMetrics = $payload['disabledMetrics'] ?? []; $scopes = \array_merge($payload['scopes'] ?? [], $scopes); if ($projectId !== $project->getId()) { @@ -122,7 +122,7 @@ class Key $scopes, $name, $expired, - $usage + $disabledMetrics ); case API_KEY_STANDARD: $key = $project->find( diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php index bed25419f6..d142e0b294 100644 --- a/src/Appwrite/Event/StatsUsage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -9,6 +9,7 @@ class StatsUsage extends Event { protected array $metrics = []; protected array $reduce = []; + protected array $disabled = []; public function __construct(protected Publisher $publisher) { @@ -41,6 +42,10 @@ class StatsUsage extends Event */ public function addMetric(string $key, int $value): self { + if ($this->disabled[$key]) { + return $this; + } + $this->metrics[] = [ 'key' => $key, 'value' => $value, @@ -49,6 +54,19 @@ class StatsUsage extends Event return $this; } + /** + * Set disabled metrics. + * + * @param string $key + * @return self + */ + public function disableMetric(string $key): self + { + $this->disabled[$key] = true; + + return $this; + } + /** * Prepare the payload for the event * diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 3c0055e637..3bf8e06356 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -196,7 +196,13 @@ class Migrations extends Action $apiKey = $jwt->encode([ 'projectId' => $project->getId(), - 'usage' => false, + 'disabledMetrics' => [ + METRIC_DATABASES_OPERATIONS_READS, + METRIC_DATABASES_OPERATIONS_WRITES, + METRIC_NETWORK_REQUESTS, + METRIC_NETWORK_INBOUND, + METRIC_NETWORK_OUTBOUND, + ], 'scopes' => [ 'users.read', 'users.write', From 3fe1147194c9eb8adb200eace559cd8a99e251b3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Feb 2025 01:07:10 +1300 Subject: [PATCH 14/21] Filter in prepare --- src/Appwrite/Event/StatsUsage.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php index d142e0b294..dd82964ae7 100644 --- a/src/Appwrite/Event/StatsUsage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -8,7 +8,7 @@ use Utopia\Queue\Publisher; class StatsUsage extends Event { protected array $metrics = []; - protected array $reduce = []; + protected array $reduce = []; protected array $disabled = []; public function __construct(protected Publisher $publisher) @@ -42,10 +42,6 @@ class StatsUsage extends Event */ public function addMetric(string $key, int $value): self { - if ($this->disabled[$key]) { - return $this; - } - $this->metrics[] = [ 'key' => $key, 'value' => $value, @@ -62,7 +58,7 @@ class StatsUsage extends Event */ public function disableMetric(string $key): self { - $this->disabled[$key] = true; + $this->disabled[] = $key; return $this; } @@ -76,8 +72,15 @@ class StatsUsage extends Event { return [ 'project' => $this->getProject(), - 'reduce' => $this->reduce, - 'metrics' => $this->metrics, + 'reduce' => $this->reduce, + 'metrics' => \array_filter($this->metrics, function ($metric) { + foreach (array_keys($this->disabled) as $disabledMetric) { + if (\str_ends_with($metric['key'], $disabledMetric)) { + return false; + } + } + return true; + }), ]; } } From 6653972e1159df15246700515dea709bb06c47f0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Feb 2025 17:29:12 +1300 Subject: [PATCH 15/21] Fix disabled metric check --- src/Appwrite/Event/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php index dd82964ae7..e259ba5e04 100644 --- a/src/Appwrite/Event/StatsUsage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -74,7 +74,7 @@ class StatsUsage extends Event 'project' => $this->getProject(), 'reduce' => $this->reduce, 'metrics' => \array_filter($this->metrics, function ($metric) { - foreach (array_keys($this->disabled) as $disabledMetric) { + foreach ($this->disabled as $disabledMetric) { if (\str_ends_with($metric['key'], $disabledMetric)) { return false; } From fe251c443fd7d42de1fef2d01d9c11f73639f7d8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Feb 2025 01:32:48 +1300 Subject: [PATCH 16/21] Remove leftover --- app/controllers/general.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 6da3354400..41a6bc5117 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -4,7 +4,6 @@ require_once __DIR__ . '/../init.php'; use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; -use Appwrite\Auth\Key; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Event\Func; @@ -771,9 +770,8 @@ App::error() ->inject('project') ->inject('logger') ->inject('log') - ->inject('apiKey') ->inject('queueForStatsUsage') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, ?Key $apiKey, StatsUsage $queueForStatsUsage) { + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, StatsUsage $queueForStatsUsage) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); From 77e83d5cfb831149dd7e9f0749f83079cb9ca1d3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Feb 2025 01:49:51 +1300 Subject: [PATCH 17/21] Remove unused injections --- app/controllers/api/databases.php | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 80d55aaabf..009116edb3 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3137,10 +3137,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('apiKey') ->inject('queueForEvents') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, ?AuthKey $apiKey, Event $queueForEvents, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3395,9 +3394,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->inject('response') ->inject('dbForProject') - ->inject('apiKey') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, ?AuthKey $apiKey, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3573,9 +3571,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->inject('response') ->inject('dbForProject') - ->inject('apiKey') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, ?AuthKey $apiKey, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3805,10 +3802,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') - ->inject('apiKey') ->inject('queueForEvents') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, ?AuthKey $apiKey, Event $queueForEvents, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -4058,10 +4054,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') - ->inject('apiKey') ->inject('queueForEvents') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, ?AuthKey $apikey, Event $queueForEvents, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); From 07a6d60b015d31a8f1882afd3c87986d6588b329 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Feb 2025 02:02:29 +1300 Subject: [PATCH 18/21] Remove leftover --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 44d66a8c02..7382d54090 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -656,7 +656,7 @@ App::shutdown() ->inject('queueForWebhooks') ->inject('queueForRealtime') ->inject('dbForProject') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, ?Key $apiKey, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) { $responsePayload = $response->getPayload(); From ff09f1e62a263b1840ff7a37076e88dd45e293dc Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Feb 2025 02:05:13 +1300 Subject: [PATCH 19/21] Remove redundant check --- tests/e2e/Services/Migrations/MigrationsBase.php | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 674f607c0a..381706f5ee 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -568,15 +568,6 @@ trait MigrationsBase $documentId = $document['body']['$id']; - // Get project stats - $initialStats = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'startDate' => UsageTest::getYesterday(), - 'endDate' => UsageTest::getTomorrow(), - ]); - $result = $this->performMigrationSync([ 'resources' => [ Resource::TYPE_DATABASE, @@ -589,9 +580,6 @@ trait MigrationsBase 'apiKey' => $this->getProject()['apiKey'], ]); - // Wait for usage dump - sleep(30); - $finalStats = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -600,10 +588,6 @@ trait MigrationsBase 'endDate' => UsageTest::getTomorrow(), ]); - // Ensure database reads/writes did not change - $this->assertEquals($initialStats['body']['databasesReadsTotal'], $finalStats['body']['databasesReadsTotal']); - $this->assertEquals($initialStats['body']['databasesWritesTotal'], $finalStats['body']['databasesWritesTotal']); - $this->assertEquals('completed', $result['status']); $this->assertEquals([Resource::TYPE_DATABASE, Resource::TYPE_COLLECTION, Resource::TYPE_ATTRIBUTE, Resource::TYPE_DOCUMENT], $result['resources']); From 0d79ae33ba947da8fea1b17cceed23ffe754bd17 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Feb 2025 02:10:05 +1300 Subject: [PATCH 20/21] Lint --- app/controllers/api/databases.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 009116edb3..59a29ac4d9 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1,7 +1,6 @@ Date: Tue, 18 Feb 2025 18:17:52 +1300 Subject: [PATCH 21/21] Fix inject --- app/controllers/shared/api.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 7382d54090..7f7b73ab0c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -644,7 +644,6 @@ App::shutdown() ->inject('response') ->inject('project') ->inject('user') - ->inject('apiKey') ->inject('queueForEvents') ->inject('queueForAudits') ->inject('queueForStatsUsage')