From 58fd0d35bd13d034c79efc2d2ee6b4210ac75955 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:47:52 +0200 Subject: [PATCH 01/26] database crud usage addition --- src/Appwrite/Event/Usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 89e900d2ab..d2e302f82b 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -69,7 +69,7 @@ class Usage extends Event 'reduce' => $this->reduce, 'metrics' => $this->metrics, ]); - + var_dump($this->metrics); $this->metrics = []; return $result; From c23af8168becf391dcc8403d10d727fe553fdeab Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:48:05 +0200 Subject: [PATCH 02/26] database crud usage addition --- app/controllers/api/databases.php | 158 +++++++++++++++++++----------- app/init.php | 2 + 2 files changed, 104 insertions(+), 56 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index aad072c50a..80bc72c4b7 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2885,7 +2885,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is not allowed for creating new documents, try update instead'); } - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('databases', $databaseId); + }); + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -2895,6 +2899,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -2981,8 +2986,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($relations as &$relation) { @@ -2995,8 +3002,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relation = new Document($relation); } if ($relation instanceof Document) { - $current = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()) + $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); + } ); if ($current->isEmpty()) { @@ -3028,6 +3037,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3037,7 +3047,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3057,8 +3067,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($related as $relation) { @@ -3116,8 +3128,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3126,6 +3141,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3155,7 +3171,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); - $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($cursorDocument->isEmpty()) { throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Document '{$documentId}' for the 'cursor' value not found."); @@ -3165,10 +3184,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { if ($document->isEmpty()) { return false; } @@ -3195,7 +3217,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); + $relatedCollection = Authorization::skip(function() use ($dbForProject, $queueForUsage, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($relations as $index => $doc) { if ($doc instanceof Document) { @@ -3274,8 +3299,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3284,7 +3312,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3293,6 +3324,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3304,7 +3336,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { if ($document->isEmpty()) { return; } @@ -3328,8 +3360,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($related as $relation) { @@ -3481,7 +3515,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('dbForProject') ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Usage $queueForUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3489,7 +3524,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DOCUMENT_MISSING_PAYLOAD); } - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('databases', $databaseId); + }); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3498,7 +3536,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3506,7 +3547,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ - $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); @@ -3548,7 +3592,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, $queueForUsage) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3570,9 +3614,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($relations as &$relation) { // If the relation is an array it can be either update or create a child document. @@ -3585,17 +3630,15 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relation = new Document($relation); } if ($relation instanceof Document) { - $oldDocument = Authorization::skip(fn () => $dbForProject->getDocument( - 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), - $relation->getId() - )); + $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); + }); + $relation->removeAttribute('$collectionId'); $relation->removeAttribute('$databaseId'); // Attribute $collection is required for Utopia. - $relation->setAttribute( - '$collection', - 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId() - ); + $relation->setAttribute('$collection', 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); if ($oldDocument->isEmpty()) { if (isset($relation['$id']) && $relation['$id'] === 'unique()') { @@ -3617,14 +3660,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); try { - $document = $dbForProject->withRequestTimestamp( - $requestTimestamp, - fn () => $dbForProject->updateDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $document->getId(), - $newDocument - ) - ); + $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); + }); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (DuplicateException) { @@ -3636,7 +3675,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3656,9 +3695,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3720,6 +3760,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3728,32 +3769,36 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Read permission should not be required for delete - $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); } try { - $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) { - $dbForProject->deleteDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $documentId - ); + $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); } catch (NotFoundException $e) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3773,9 +3818,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($related as $relation) { if ($relation instanceof Document) { diff --git a/app/init.php b/app/init.php index c6847610c2..d5b8e4b235 100644 --- a/app/init.php +++ b/app/init.php @@ -232,6 +232,8 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; +const METRIC_DATABASE_API_READ = 'database.api.events.read'; +const METRIC_DATABASE_API_WRITE = 'database.api.events.write'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From 025c9aeeb297f6a0a96df401c6d0281321d17fb5 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:53:45 +0200 Subject: [PATCH 03/26] database crud usage addition --- src/Appwrite/Event/Usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index d2e302f82b..89e900d2ab 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -69,7 +69,7 @@ class Usage extends Event 'reduce' => $this->reduce, 'metrics' => $this->metrics, ]); - var_dump($this->metrics); + $this->metrics = []; return $result; From 8e17fc69b371861ee04da7e98a99fac294b42588 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:57:29 +0200 Subject: [PATCH 04/26] database crud usage addition --- app/controllers/api/databases.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 80bc72c4b7..b98df1382f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2898,8 +2898,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); From dd13e8b3cf705be22540b93ff3be493a771fc399 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 18:02:19 +0200 Subject: [PATCH 05/26] database crud usage addition --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b98df1382f..4275ab52ae 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3219,7 +3219,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function() use ($dbForProject, $queueForUsage, $database, $relatedCollectionId) { + $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); From 0f2985cec1a2e80fc325b59824eedad64519b5f2 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:19:09 +0200 Subject: [PATCH 06/26] database crud usage addition --- app/controllers/api/databases.php | 56 +++++++++++++++---------------- app/init.php | 4 +-- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 4275ab52ae..e51201a467 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2886,7 +2886,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('databases', $databaseId); }); @@ -2899,7 +2899,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -2989,7 +2989,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3005,7 +3005,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } if ($relation instanceof Document) { $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); } ); @@ -3039,7 +3039,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3070,7 +3070,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3133,7 +3133,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3143,7 +3143,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3174,7 +3174,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3186,10 +3186,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); // Add $collectionId and $databaseId for all documents $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { @@ -3220,7 +3220,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3305,7 +3305,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3315,7 +3315,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3326,7 +3326,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3363,7 +3363,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3527,7 +3527,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('databases', $databaseId); }); @@ -3539,7 +3539,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3550,7 +3550,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3617,7 +3617,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3633,7 +3633,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } if ($relation instanceof Document) { $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); }); @@ -3663,7 +3663,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum try { $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); }); } catch (AuthorizationException) { @@ -3698,7 +3698,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3762,7 +3762,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3772,7 +3772,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3782,7 +3782,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu // Read permission should not be required for delete $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3792,7 +3792,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu try { $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); } catch (NotFoundException $e) { @@ -3821,7 +3821,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); diff --git a/app/init.php b/app/init.php index d5b8e4b235..d942e056db 100644 --- a/app/init.php +++ b/app/init.php @@ -232,8 +232,8 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASE_API_READ = 'database.api.events.read'; -const METRIC_DATABASE_API_WRITE = 'database.api.events.write'; +const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From 42589de95ef41114d247fdbceacce8ea3d708233 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:56:14 +0200 Subject: [PATCH 07/26] database crud usage addition --- app/controllers/api/databases.php | 170 +++++++++++++----------------- app/init.php | 2 + 2 files changed, 75 insertions(+), 97 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index e51201a467..ea17065d45 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2885,11 +2885,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is not allowed for creating new documents, try update instead'); } - $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('databases', $databaseId); - }); - + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -2898,10 +2894,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -2988,10 +2981,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($relations as &$relation) { @@ -3004,10 +2995,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relation = new Document($relation); } if ($relation instanceof Document) { - $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); - } + $current = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()) ); if ($current->isEmpty()) { @@ -3039,7 +3028,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3049,7 +3037,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3069,10 +3057,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($related as $relation) { @@ -3107,6 +3093,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); @@ -3133,7 +3121,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3143,7 +3130,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3173,10 +3159,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); - $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($cursorDocument->isEmpty()) { throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Document '{$documentId}' for the 'cursor' value not found."); @@ -3186,13 +3169,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { if ($document->isEmpty()) { return false; } @@ -3219,10 +3199,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); foreach ($relations as $index => $doc) { if ($doc instanceof Document) { @@ -3273,6 +3250,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ; + $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3305,7 +3287,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3314,10 +3295,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3326,7 +3304,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3338,7 +3315,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { if ($document->isEmpty()) { return; } @@ -3362,10 +3339,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($related as $relation) { @@ -3378,6 +3353,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3526,10 +3506,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DOCUMENT_MISSING_PAYLOAD); } - $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('databases', $databaseId); - }); + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3538,10 +3515,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3549,10 +3523,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ - $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); @@ -3594,7 +3565,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, $queueForUsage) { + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3616,10 +3587,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($relations as &$relation) { // If the relation is an array it can be either update or create a child document. @@ -3632,15 +3602,17 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relation = new Document($relation); } if ($relation instanceof Document) { - $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); - }); - + $oldDocument = Authorization::skip(fn () => $dbForProject->getDocument( + 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), + $relation->getId() + )); $relation->removeAttribute('$collectionId'); $relation->removeAttribute('$databaseId'); // Attribute $collection is required for Utopia. - $relation->setAttribute('$collection', 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + $relation->setAttribute( + '$collection', + 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId() + ); if ($oldDocument->isEmpty()) { if (isset($relation['$id']) && $relation['$id'] === 'unique()') { @@ -3662,10 +3634,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); try { - $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); - return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); - }); + $document = $dbForProject->withRequestTimestamp( + $requestTimestamp, + fn () => $dbForProject->updateDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $document->getId(), + $newDocument + ) + ); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (DuplicateException) { @@ -3677,7 +3653,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3697,10 +3673,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3712,6 +3687,11 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( @@ -3762,7 +3742,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3771,36 +3750,32 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Read permission should not be required for delete - $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); } try { - $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); - $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) { + $dbForProject->deleteDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $documentId + ); }); } catch (NotFoundException $e) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3820,10 +3795,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3852,6 +3826,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->noContent(); diff --git a/app/init.php b/app/init.php index d942e056db..cf41bc8049 100644 --- a/app/init.php +++ b/app/init.php @@ -233,7 +233,9 @@ const API_KEY_DYNAMIC = 'dynamic'; const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From c7841bb36ef0b503a6d1001ecfc78ca8ea360e44 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:59:59 +0200 Subject: [PATCH 08/26] database crud usage addition --- app/controllers/api/databases.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index ea17065d45..0ab84bf0fc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3121,7 +3121,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3285,9 +3284,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('mode') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); From fbbc9312eaf3ce85042efd74eb09f3e708394d7f Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:09:44 +0200 Subject: [PATCH 09/26] database crud usage addition --- app/controllers/api/databases.php | 61 +++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0ab84bf0fc..1249ccb2fa 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3036,8 +3036,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3049,6 +3051,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3093,8 +3099,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); @@ -3170,8 +3176,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { if ($document->isEmpty()) { return false; } @@ -3188,6 +3196,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3198,6 +3210,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); + // todo: Use local cache for this getDocument $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); foreach ($relations as $index => $doc) { @@ -3250,8 +3263,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->dynamic(new Document([ @@ -3311,8 +3324,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { if ($document->isEmpty()) { return; } @@ -3328,6 +3343,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3351,8 +3370,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -3649,8 +3668,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3662,6 +3683,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3685,8 +3710,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -3771,8 +3796,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3784,6 +3811,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3823,8 +3854,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->noContent(); From c0d48efe18b7ea40f654a7303c7f2192a4e5bf60 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:29:55 +0200 Subject: [PATCH 10/26] composer --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 1249ccb2fa..a0fea5e293 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); From e7e095907cdf58a2a5627109f9b164e60a34e021 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:44:56 +0200 Subject: [PATCH 11/26] composer --- app/controllers/api/databases.php | 4 ++-- app/init.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a0fea5e293..1249ccb2fa 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); diff --git a/app/init.php b/app/init.php index cf41bc8049..2f83e7c3b2 100644 --- a/app/init.php +++ b/app/init.php @@ -232,9 +232,9 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; -const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; From 504b7276cb3fc0c94967fd177151a629e629b50b Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 11:02:53 +0200 Subject: [PATCH 12/26] composer --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 1249ccb2fa..a0fea5e293 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); From 63cf093020451c8fba1668443e4a5432621d2ebe Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 13:53:57 +0200 Subject: [PATCH 13/26] file transformations usage --- app/controllers/api/storage.php | 15 ++++++++++++++- app/init.php | 10 ++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 7d4361a192..97fb5cc711 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -6,6 +6,7 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Database\Validator\CustomId; @@ -886,7 +887,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal) { + ->inject('queueForUsage') + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, Usage $queueForUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1014,6 +1016,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; + foreach([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $param){ + if(!empty($param)){ + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; + break; + } + } + + $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days ->setContentType($contentType) diff --git a/app/init.php b/app/init.php index 2f83e7c3b2..5e25fa905f 100644 --- a/app/init.php +++ b/app/init.php @@ -232,10 +232,6 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; -const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; -const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; -const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; @@ -263,9 +259,15 @@ const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; +const METRIC_FILES_TRANSFORMATIONS = 'files.transformations'; +const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; const METRIC_FUNCTIONS = 'functions'; From 6b45293f7828f36ee590019a1593c84a6b175747 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 13:55:30 +0200 Subject: [PATCH 14/26] file transformations usage --- app/controllers/api/storage.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 97fb5cc711..8431fb19e3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1016,14 +1016,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - foreach([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $param){ - if(!empty($param)){ - $queueForUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - break; - } + foreach ([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $parameter) { + if (!empty($parameter)) { + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; + break; + } } From c5a49cc5375ab4fdc512fe9207802bd6f4fe18df Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 16:45:21 +0200 Subject: [PATCH 15/26] added operations debug header --- app/controllers/api/databases.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a0fea5e293..c3f74389cc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3077,6 +3077,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + + $response->addHeader('X-Debug-Operations', $operations); $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3098,10 +3105,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setPayload($response->getPayload(), sensitive: $relationships); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3267,6 +3270,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3374,6 +3378,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3714,6 +3719,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( @@ -3858,6 +3864,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 4acc2736e40a21b1c6ecfb9e97b1298086888b25 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 16:46:38 +0200 Subject: [PATCH 16/26] added operations debug header --- app/controllers/api/storage.php | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8431fb19e3..e92b55f140 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1016,16 +1016,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - foreach ([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $parameter) { - if (!empty($parameter)) { - $queueForUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - break; - } - } - + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days From 1a73847c4798e77923da52a1f231f9fc7bb0cf70 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 11:28:26 +0200 Subject: [PATCH 17/26] added operations debug header --- app/controllers/api/databases.php | 67 ++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c3f74389cc..16258e9972 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,7 +2944,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database) { + $operations = 1; + + $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -2976,10 +2978,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') if ($isList) { $relations = $related; + $operations += count($related); } else { $relations = [$related]; + $operations++; } + $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip( fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) @@ -3026,6 +3031,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ; + try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3036,7 +3046,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3078,8 +3088,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3238,6 +3248,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ; + $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); }, false); @@ -3265,11 +3280,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; - $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, @@ -3586,7 +3596,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { + + $operations = 1; + + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3595,6 +3608,13 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + /** + * Write relationship counts + */ + if (\in_array(\gettype($related), ['array', 'object'])) { + + } + if (empty($related)) { continue; } @@ -3603,8 +3623,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum if ($isList) { $relations = $related; + $operations += count($related); } else { $relations = [$related]; + $operations++; } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); @@ -3654,6 +3676,11 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ; + try { $document = $dbForProject->withRequestTimestamp( $requestTimestamp, @@ -3673,7 +3700,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3715,8 +3742,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->addHeader('X-Debug-Operations', $operations); @@ -3843,6 +3870,15 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + + $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( @@ -3859,11 +3895,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 895e79e7369715be206e01cfdf8fe3eca3376450 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 12:25:56 +0200 Subject: [PATCH 18/26] Operations --- app/controllers/api/databases.php | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 16258e9972..ac41f2d2fe 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,7 +2944,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; + $operations = 1; // For root createDocument $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); @@ -3046,7 +3046,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; + $operations = 0; // Since the Document is sent through post no getDocument used // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3597,7 +3597,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); - $operations = 1; + $operations = 1; // Root updateDocument $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( @@ -3700,7 +3700,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; + $operations = 1; // Since we read original Document // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3811,6 +3811,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; // Read original Document + // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3829,8 +3831,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; - // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); @@ -3877,8 +3877,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - - $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( From 0814fe49e204221ac86f021f325cc1c0fd92cd25 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 16:50:23 +0200 Subject: [PATCH 19/26] added operations debug header --- app/controllers/api/databases.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 16258e9972..8f59033795 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3031,11 +3031,14 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); + $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-operations-write', $operations); + try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3092,8 +3095,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-read', $operations); $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3253,6 +3255,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-operations-reads', $operations); + $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); }, false); @@ -3280,7 +3284,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } - $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3388,7 +3391,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-reads', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3681,6 +3684,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-operations-writes', $operations); + try { $document = $dbForProject->withRequestTimestamp( $requestTimestamp, @@ -3746,7 +3751,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-reads', $operations); + $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( From f70222d41a532d00bb2db34a767bc64c7f2e96f4 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 17:19:39 +0200 Subject: [PATCH 20/26] added operations debug header --- app/controllers/api/databases.php | 35 ++++++++++--------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 3f8fe7cfdc..fa12087463 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,8 +2944,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; // For root createDocument - + $operations = 1; $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -3031,14 +3030,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); - - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ; - - $response->addHeader('X-Debug-operations-write', $operations); - try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3049,10 +3040,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; // Since the Document is sent through post no getDocument used // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3064,10 +3054,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3091,11 +3077,12 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-operations-read', $operations); + $response->addHeader('X-Debug-Operations', $operations); + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3255,7 +3242,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-operations-reads', $operations); + $response->addHeader('X-Debug-Operations', $operations); $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); @@ -3391,7 +3378,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-operations-reads', $operations); + $response->addHeader('X-Debug-Operations', $operations); + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3600,8 +3588,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); - $operations = 1; // Root updateDocument - + $operations = 1; $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3684,7 +3671,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; - $response->addHeader('X-Debug-operations-writes', $operations); + $response->addHeader('X-Debug-Operations', $operations); try { $document = $dbForProject->withRequestTimestamp( From fc03075ef895d4818029445d6af32fd55cfd4ae0 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 18:01:44 +0200 Subject: [PATCH 21/26] Update counts --- app/controllers/api/databases.php | 28 +++++++++++--------- composer.lock | 44 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index fa12087463..f6de0d7d0f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3163,7 +3163,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); } - $documentId = $cursor->getValue(); $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3178,7 +3177,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { @@ -3186,6 +3185,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') return false; } + $operations++; + $document->removeAttribute('$collection'); $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3198,13 +3199,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { + if (\gettype($related) === 'object'){ $operations++; + } else if(\gettype($related) === 'array'){ + $operations += max(1, count($related)); } if (empty($related)) { continue; } + if (!\is_array($related)) { $relations = [$related]; } else { @@ -3328,7 +3332,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3336,6 +3340,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen return; } + $operations++; + $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3347,13 +3353,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { + if (\gettype($related) === 'object'){ $operations++; + } else if(\gettype($related) === 'array'){ + $operations += max(1, count($related)); } if (empty($related)) { continue; } + if (!\is_array($related)) { $related = [$related]; } @@ -3587,8 +3596,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $operations = 1; + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3598,13 +3607,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - /** - * Write relationship counts - */ - if (\in_array(\gettype($related), ['array', 'object'])) { - - } - if (empty($related)) { continue; } diff --git a/composer.lock b/composer.lock index ce5f358619..dbe9e079b5 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.1", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" + "reference": "339ba21476eb184290361542f732ad12c97591ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", + "reference": "339ba21476eb184290361542f732ad12c97591ec", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.1" + "source": "https://github.com/symfony/http-client/tree/v7.2.2" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:50:44+00:00" + "time": "2024-12-30T18:35:15+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.18.3", + "version": "v1.19.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", + "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.65.0", - "illuminate/view": "^10.48.24", - "larastan/larastan": "^2.9.11", - "laravel-zero/framework": "^10.4.0", + "friendsofphp/php-cs-fixer": "^3.66.0", + "illuminate/view": "^10.48.25", + "larastan/larastan": "^2.9.12", + "laravel-zero/framework": "^10.48.25", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-11-26T15:34:00+00:00" + "time": "2024-12-30T16:20:10+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.0", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" + "reference": "87a71856f2f56e4100373e92529eed3171695cfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", + "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", + "reference": "87a71856f2f56e4100373e92529eed3171695cfb", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.0" + "source": "https://github.com/symfony/finder/tree/v7.2.2" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-10-23T06:56:12+00:00" + "time": "2024-12-30T19:00:17+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From d64ea97ee6ff6f895477cb4b54166e9f6261c0e3 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 18:06:18 +0200 Subject: [PATCH 22/26] added operations debug header --- app/controllers/api/databases.php | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index fa12087463..c65c76297d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3692,10 +3692,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Since we read original Document - // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3707,10 +3705,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3733,13 +3727,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; - - $response->addHeader('X-Debug-operations-reads', $operations); - $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( From 40b407027efb1ddcb453b6570cd033ed004ff717 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 18:51:39 +0200 Subject: [PATCH 23/26] Count changes --- app/controllers/api/databases.php | 49 ++++++++++++------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b9614f5ef9..ca81519834 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,8 +2944,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; + $operations = 0; + $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { + $operations++; + $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -2977,13 +2980,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') if ($isList) { $relations = $related; - $operations += count($related); } else { $relations = [$related]; - $operations++; } - $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip( fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) @@ -3102,8 +3102,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('collection', $collection) ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); - - }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3199,13 +3197,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\gettype($related) === 'object'){ - $operations++; - } else if(\gettype($related) === 'array'){ - $operations += max(1, count($related)); - } - if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } @@ -3353,13 +3349,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\gettype($related) === 'object'){ - $operations++; - } else if(\gettype($related) === 'array'){ - $operations += max(1, count($related)); - } - if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } @@ -3596,9 +3590,12 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $operations = 1; + $operations = 0; $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { + + $operations++; + $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3615,10 +3612,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum if ($isList) { $relations = $related; - $operations += count($related); } else { $relations = [$related]; - $operations++; } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); @@ -3793,8 +3788,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Read original Document - // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3814,7 +3807,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3826,10 +3819,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3855,8 +3844,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $relationships = \array_map( @@ -3875,7 +3862,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-Operations', 1); $response->noContent(); }); From a481e958e345b220a98b252277043cd4de8eac47 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 18:57:35 +0200 Subject: [PATCH 24/26] added operations debug header --- app/controllers/api/databases.php | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b9614f5ef9..5503a78051 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3078,7 +3078,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->addHeader('X-Debug-Operations', $operations); @@ -3102,8 +3102,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('collection', $collection) ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); - - }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3177,9 +3175,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $operations = 0; - // Add $collectionId and $databaseId for all documents + $operations = 1; $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { if ($document->isEmpty()) { return false; @@ -3332,9 +3329,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } - $operations = 0; - // Add $collectionId and $databaseId for all documents + $operations = 1; $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { if ($document->isEmpty()) { return; @@ -3491,6 +3487,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); } } + $response->dynamic(new Document([ 'total' => $audit->countLogsByResource($resource), 'logs' => $output, @@ -3597,7 +3594,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); $operations = 1; - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3793,8 +3789,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Read original Document - // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3814,7 +3808,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3826,10 +3820,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3855,10 +3845,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->addHeader('X-Debug-Operations', $operations); + $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( @@ -3875,7 +3865,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 5c097df56a1710925fbb3bbe0973380147975b21 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 19:07:18 +0200 Subject: [PATCH 25/26] added operations debug header --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 92e61ad1d5..347b4ebbef 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3847,7 +3847,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-Operations', 1); $relationships = \array_map( fn ($document) => $document->getAttribute('key'), From f2cbdbae870babc3c92fca3d10d0c4a255e4ecdf Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 19:32:19 +0200 Subject: [PATCH 26/26] added operations debug header --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index dbe9e079b5..b906af3a81 100644 --- a/composer.lock +++ b/composer.lock @@ -8580,5 +8580,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" }