diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index aad072c50a..347b4ebbef 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,7 +2944,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database) { + $operations = 0; + + $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { + $operations++; + $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -3036,6 +3040,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } + // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); @@ -3071,6 +3076,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + $response->addHeader('X-Debug-Operations', $operations); + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3090,10 +3102,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('collection', $collection) ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); - - - $queueForUsage - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3116,7 +3124,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3152,7 +3161,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); } - $documentId = $cursor->getValue(); $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3167,12 +3175,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); + $operations = 0; + // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { if ($document->isEmpty()) { return false; } + $operations++; + $document->removeAttribute('$collection'); $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3186,8 +3198,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $related = $document->getAttribute($relationship->getAttribute('key')); if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } + if (!\is_array($related)) { $relations = [$related]; } else { @@ -3195,6 +3212,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); + // todo: Use local cache for this getDocument $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); foreach ($relations as $index => $doc) { @@ -3219,6 +3237,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ; + + $response->addHeader('X-Debug-Operations', $operations); + $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); }, false); @@ -3274,9 +3299,9 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3303,12 +3328,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } + $operations = 0; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { if ($document->isEmpty()) { return; } + $operations++; + $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3321,8 +3350,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $related = $document->getAttribute($relationship->getAttribute('key')); if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } + if (!\is_array($related)) { $related = [$related]; } @@ -3342,6 +3376,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ; + + $response->addHeader('X-Debug-Operations', $operations); + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3444,6 +3485,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); } } + $response->dynamic(new Document([ 'total' => $audit->countLogsByResource($resource), 'logs' => $output, @@ -3481,7 +3523,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('dbForProject') ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Usage $queueForUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3548,7 +3591,12 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { + $operations = 0; + + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { + + $operations++; + $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3616,6 +3664,13 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ; + + $response->addHeader('X-Debug-Operations', $operations); + try { $document = $dbForProject->withRequestTimestamp( $requestTimestamp, @@ -3787,6 +3842,13 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + $response->addHeader('X-Debug-Operations', 1); + $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( @@ -3803,9 +3865,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $queueForUsage - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->noContent(); }); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 7d4361a192..e92b55f140 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -6,6 +6,7 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Database\Validator\CustomId; @@ -886,7 +887,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal) { + ->inject('queueForUsage') + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, Usage $queueForUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1014,6 +1016,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; + $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days ->setContentType($contentType) diff --git a/app/init.php b/app/init.php index 7f95ac4c25..b4e60d7924 100644 --- a/app/init.php +++ b/app/init.php @@ -259,9 +259,15 @@ const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; +const METRIC_FILES_TRANSFORMATIONS = 'files.transformations'; +const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; const METRIC_FUNCTIONS = 'functions'; diff --git a/composer.lock b/composer.lock index ce5f358619..b906af3a81 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.1", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" + "reference": "339ba21476eb184290361542f732ad12c97591ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", + "reference": "339ba21476eb184290361542f732ad12c97591ec", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.1" + "source": "https://github.com/symfony/http-client/tree/v7.2.2" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:50:44+00:00" + "time": "2024-12-30T18:35:15+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.18.3", + "version": "v1.19.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", + "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.65.0", - "illuminate/view": "^10.48.24", - "larastan/larastan": "^2.9.11", - "laravel-zero/framework": "^10.4.0", + "friendsofphp/php-cs-fixer": "^3.66.0", + "illuminate/view": "^10.48.25", + "larastan/larastan": "^2.9.12", + "laravel-zero/framework": "^10.48.25", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-11-26T15:34:00+00:00" + "time": "2024-12-30T16:20:10+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.0", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" + "reference": "87a71856f2f56e4100373e92529eed3171695cfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", + "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", + "reference": "87a71856f2f56e4100373e92529eed3171695cfb", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.0" + "source": "https://github.com/symfony/finder/tree/v7.2.2" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-10-23T06:56:12+00:00" + "time": "2024-12-30T19:00:17+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8580,5 +8580,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" }