diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d3a7d060b0..11966d14f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -319,10 +319,16 @@ These are the current metrics we collect usage stats for: | users | Total number of users per project| | executions | Total number of executions per project | | databases | Total number of databases per project | +| databases.storage | Total amount of storage used by all databases per project (in bytes) | +| databases.storage_disk | Total amount of storage used by all database per project on disk (in bytes) | | collections | Total number of collections per project | | {databaseInternalId}.collections | Total number of collections per database| +| {databaseInternalId}.storage | Sum of database storage (in bytes) | +| {databaseInternalId}.storage_disk | Sum of database storage on disk (in bytes) | | documents | Total number of documents per project | | {databaseInternalId}.{collectionInternalId}.documents | Total number of documents per collection | +| {databaseInternalId}.{collectionInternalId}.storage | Sum of database storage used by the collection (in bytes) | +| {databsaeInternalId}.{collectionInternalId}.storage_disk | Sum of database storage used by the collection on disk (in bytes) | | buckets | Total number of buckets per project | | files | Total number of files per project | | {bucketInternalId}.files.storage | Sum of files.storage per bucket (in bytes) | diff --git a/app/config/specs/open-api3-1.6.x-console.json b/app/config/specs/open-api3-1.6.x-console.json index 07749889d8..1bd2adf6b5 100644 --- a/app/config/specs/open-api3-1.6.x-console.json +++ b/app/config/specs/open-api3-1.6.x-console.json @@ -36747,6 +36747,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -36770,6 +36776,14 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ @@ -36777,9 +36791,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -36803,6 +36819,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -36818,14 +36840,24 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -37366,6 +37398,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -37462,6 +37500,14 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "$ref": "#\/components\/schemas\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -37491,6 +37537,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -37505,6 +37552,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 07749889d8..1bd2adf6b5 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -36747,6 +36747,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -36770,6 +36776,14 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ @@ -36777,9 +36791,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -36803,6 +36819,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -36818,14 +36840,24 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -37366,6 +37398,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -37462,6 +37500,14 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "$ref": "#\/components\/schemas\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -37491,6 +37537,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -37505,6 +37552,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/config/specs/swagger2-1.6.x-console.json b/app/config/specs/swagger2-1.6.x-console.json index 51935a5e01..71721d9ac6 100644 --- a/app/config/specs/swagger2-1.6.x-console.json +++ b/app/config/specs/swagger2-1.6.x-console.json @@ -37270,6 +37270,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -37296,6 +37302,15 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ @@ -37303,9 +37318,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -37329,6 +37346,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -37346,14 +37369,25 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -37921,6 +37955,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -38023,6 +38063,15 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -38055,6 +38104,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -38069,6 +38119,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 51935a5e01..71721d9ac6 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -37270,6 +37270,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -37296,6 +37302,15 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ @@ -37303,9 +37318,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -37329,6 +37346,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -37346,14 +37369,25 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -37921,6 +37955,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -38023,6 +38063,15 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -38055,6 +38104,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -38069,6 +38119,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a9bb58df4b..b76548b725 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -5,6 +5,7 @@ use Appwrite\Detector\Detector; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email; use Appwrite\Utopia\Database\Validator\CustomId; @@ -452,7 +453,8 @@ App::post('/v1/databases') ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage) { $databaseId = $databaseId == 'unique()' ? ID::unique() : $databaseId; @@ -502,6 +504,7 @@ App::post('/v1/databases') } $queueForEvents->setParam('databaseId', $database->getId()); + $queueForUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -733,7 +736,8 @@ App::delete('/v1/databases/:databaseId') ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { + ->inject('queueForUsage') + ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -756,6 +760,9 @@ App::delete('/v1/databases/:databaseId') ->setParam('databaseId', $database->getId()) ->setPayload($response->output($database, Response::MODEL_DATABASE)); + $queueForUsage + ->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation + $response->noContent(); }); @@ -2350,7 +2357,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) { $db = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -2435,6 +2443,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->setContext('database', $db) ->setPayload($response->output($attribute, $model)); + $queueForUsage + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->noContent(); }); @@ -2810,8 +2821,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') + ->inject('queueForUsage') ->inject('mode') - ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode) { + ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3027,6 +3039,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); + + $queueForUsage + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3643,8 +3658,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('dbForProject') ->inject('queueForDeletes') ->inject('queueForEvents') + ->inject('queueForUsage') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -3729,6 +3745,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); + $queueForUsage + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->noContent(); }); @@ -3754,6 +3773,7 @@ App::get('/v1/databases/usage') METRIC_DATABASES, METRIC_COLLECTIONS, METRIC_DOCUMENTS, + METRIC_DATABASES_STORAGE ]; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { @@ -3804,9 +3824,11 @@ App::get('/v1/databases/usage') 'databasesTotal' => $usage[$metrics[0]]['total'], 'collectionsTotal' => $usage[$metrics[1]]['total'], 'documentsTotal' => $usage[$metrics[2]]['total'], + 'storageTotal' => $usage[$metrics[3]]['total'], 'databases' => $usage[$metrics[0]]['data'], 'collections' => $usage[$metrics[1]]['data'], 'documents' => $usage[$metrics[2]]['data'], + 'storage' => $usage[$metrics[3]]['data'], ]), Response::MODEL_USAGE_DATABASES); }); @@ -3838,6 +3860,7 @@ App::get('/v1/databases/:databaseId/usage') $metrics = [ str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS), str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS), + str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE) ]; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { @@ -3888,8 +3911,10 @@ App::get('/v1/databases/:databaseId/usage') 'range' => $range, 'collectionsTotal' => $usage[$metrics[0]]['total'], 'documentsTotal' => $usage[$metrics[1]]['total'], + 'storageTotal' => $usage[$metrics[2]]['total'], 'collections' => $usage[$metrics[0]]['data'], 'documents' => $usage[$metrics[1]]['data'], + 'storage' => $usage[$metrics[2]]['data'], ]), Response::MODEL_USAGE_DATABASE); }); diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 62a34bb5ce..6053326308 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -47,6 +47,7 @@ App::get('/v1/project/usage') METRIC_USERS, METRIC_BUCKETS, METRIC_FILES_STORAGE, + METRIC_DATABASES_STORAGE, METRIC_DEPLOYMENTS_STORAGE, METRIC_BUILDS_STORAGE ], @@ -56,6 +57,7 @@ App::get('/v1/project/usage') METRIC_NETWORK_OUTBOUND, METRIC_USERS, METRIC_EXECUTIONS, + METRIC_DATABASES_STORAGE, METRIC_EXECUTIONS_MB_SECONDS, METRIC_BUILDS_MB_SECONDS ] @@ -182,6 +184,23 @@ App::get('/v1/project/usage') ]; }, $dbForProject->find('buckets')); + $databasesStorageBreakdown = array_map(function ($database) use ($dbForProject) { + $id = $database->getId(); + $name = $database->getAttribute('name'); + $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE); + + $value = $dbForProject->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + return [ + 'resourceId' => $id, + 'name' => $name, + 'value' => $value['value'] ?? 0, + ]; + }, $dbForProject->find('databases')); + $functionsStorageBreakdown = array_map(function ($function) use ($dbForProject) { $id = $function->getId(); $name = $function->getAttribute('name'); @@ -269,6 +288,7 @@ App::get('/v1/project/usage') 'buildsMbSecondsTotal' => $total[METRIC_BUILDS_MB_SECONDS], 'documentsTotal' => $total[METRIC_DOCUMENTS], 'databasesTotal' => $total[METRIC_DATABASES], + 'databasesStorageTotal' => $total[METRIC_DATABASES_STORAGE], 'usersTotal' => $total[METRIC_USERS], 'bucketsTotal' => $total[METRIC_BUCKETS], 'filesStorageTotal' => $total[METRIC_FILES_STORAGE], @@ -279,6 +299,7 @@ App::get('/v1/project/usage') 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, + 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, 'functionsStorageBreakdown' => $functionsStorageBreakdown, diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 6d87940ff7..357d73adc8 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -95,7 +95,7 @@ $databaseListener = function (string $event, Document $document, Document $proje $databaseInternalId = $parts[1] ?? 0; $queueForUsage ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) ; if ($event === Database::EVENT_DOCUMENT_DELETE) { diff --git a/app/init.php b/app/init.php index b4ab772e0e..f94c9665e7 100644 --- a/app/init.php +++ b/app/init.php @@ -238,10 +238,16 @@ const METRIC_MESSAGES_TYPE_PROVIDER_FAILED = METRIC_MESSAGES . '.{type}.{provid const METRIC_SESSIONS = 'sessions'; const METRIC_DATABASES = 'databases'; const METRIC_COLLECTIONS = 'collections'; +const METRIC_DATABASES_STORAGE = 'databases.storage'; +const METRIC_DATABASES_STORAGE_DISK = 'databases.storage_disk'; const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE = '{databaseInternalId}.databases.storage'; +const METRIC_DATABASE_ID_STORAGE_DISK = '{databaseInternalId}.databases.storage_disk'; const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE_DISK = '{databaseInternalId}.{collectionInternalId}.databases.storage_disk'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 8f83fed544..ad35135a6f 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -531,6 +531,8 @@ $image = $this->getParam('image', ''); - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_LOGGING_CONFIG + - _APP_DOMAIN + - _APP_OPTIONS_FORCE_HTTPS appwrite-worker-messaging: image: /: diff --git a/composer.json b/composer.json index 91ff1eeb92..26b8c3a7a3 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.10.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.53.*", + "utopia-php/database": "0.53.5-rc1", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", @@ -97,5 +97,11 @@ "platform": { "php": "8.3" } + }, + "repositories": { + "utopia-php/database": { + "type": "vcs", + "url": "https://github.com/utopia-php/database" + } } } diff --git a/composer.lock b/composer.lock index 147800df32..b3a94545b4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b6820da26239716cf14a445697902a03", + "content-hash": "7066d9ca32e7a1a60614effdc4701970", "packages": [ { "name": "adhocore/jwt", @@ -1723,16 +1723,16 @@ }, { "name": "utopia-php/database", - "version": "0.53.4", + "version": "0.53.5-rc1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "36a0e89d983afc1368635282e04fa762220a1d2a" + "reference": "689ba22063bf46def385da8695ba7a921e81e38d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/36a0e89d983afc1368635282e04fa762220a1d2a", - "reference": "36a0e89d983afc1368635282e04fa762220a1d2a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/689ba22063bf46def385da8695ba7a921e81e38d", + "reference": "689ba22063bf46def385da8695ba7a921e81e38d", "shasum": "" }, "require": { @@ -1759,7 +1759,38 @@ "Utopia\\Database\\": "src/Database" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Tests\\E2E\\": "tests/e2e", + "Tests\\Unit\\": "tests/unit" + } + }, + "scripts": { + "build": [ + "Composer\\Config::disableProcessTimeout", + "docker compose build" + ], + "start": [ + "Composer\\Config::disableProcessTimeout", + "docker compose up -d" + ], + "test": [ + "Composer\\Config::disableProcessTimeout", + "docker compose exec tests vendor/bin/phpunit --configuration phpunit.xml" + ], + "lint": [ + "./vendor/bin/pint --test" + ], + "format": [ + "./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 7 src tests --memory-limit 512M" + ], + "coverage": [ + "./vendor/bin/coverage-check ./tmp/clover.xml 90" + ] + }, "license": [ "MIT" ], @@ -1772,10 +1803,10 @@ "utopia" ], "support": { - "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.53.4" + "source": "https://github.com/utopia-php/database/tree/0.53.5-rc1", + "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2024-09-10T10:19:57+00:00" + "time": "2024-09-24T08:43:10+00:00" }, { "name": "utopia-php/domains", @@ -2069,16 +2100,16 @@ }, { "name": "utopia-php/logger", - "version": "0.6.0", + "version": "0.6.1", "source": { "type": "git", "url": "https://github.com/utopia-php/logger.git", - "reference": "a2d1daeeb8f61fdec6d851950d9a021a3d05c9f9" + "reference": "7e8ff512c6f04577aba1df67c7b9628971946f9c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/logger/zipball/a2d1daeeb8f61fdec6d851950d9a021a3d05c9f9", - "reference": "a2d1daeeb8f61fdec6d851950d9a021a3d05c9f9", + "url": "https://api.github.com/repos/utopia-php/logger/zipball/7e8ff512c6f04577aba1df67c7b9628971946f9c", + "reference": "7e8ff512c6f04577aba1df67c7b9628971946f9c", "shasum": "" }, "require": { @@ -2117,9 +2148,9 @@ ], "support": { "issues": "https://github.com/utopia-php/logger/issues", - "source": "https://github.com/utopia-php/logger/tree/0.6.0" + "source": "https://github.com/utopia-php/logger/tree/0.6.1" }, - "time": "2024-05-23T13:37:54+00:00" + "time": "2024-09-20T14:02:12+00:00" }, { "name": "utopia-php/messaging", @@ -4185,16 +4216,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.30.1", + "version": "1.31.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "51b95ec8670af41009e2b2b56873bad96682413e" + "reference": "249f15fb843bf240cf058372dad29e100cee6c17" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/51b95ec8670af41009e2b2b56873bad96682413e", - "reference": "51b95ec8670af41009e2b2b56873bad96682413e", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/249f15fb843bf240cf058372dad29e100cee6c17", + "reference": "249f15fb843bf240cf058372dad29e100cee6c17", "shasum": "" }, "require": { @@ -4226,9 +4257,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.30.1" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.31.0" }, - "time": "2024-09-07T20:13:05+00:00" + "time": "2024-09-22T11:32:18+00:00" }, { "name": "phpunit/php-code-coverage", @@ -5865,16 +5896,16 @@ }, { "name": "symfony/console", - "version": "v7.1.4", + "version": "v7.1.5", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "1eed7af6961d763e7832e874d7f9b21c3ea9c111" + "reference": "0fa539d12b3ccf068a722bbbffa07ca7079af9ee" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/1eed7af6961d763e7832e874d7f9b21c3ea9c111", - "reference": "1eed7af6961d763e7832e874d7f9b21c3ea9c111", + "url": "https://api.github.com/repos/symfony/console/zipball/0fa539d12b3ccf068a722bbbffa07ca7079af9ee", + "reference": "0fa539d12b3ccf068a722bbbffa07ca7079af9ee", "shasum": "" }, "require": { @@ -5938,7 +5969,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.1.4" + "source": "https://github.com/symfony/console/tree/v7.1.5" }, "funding": [ { @@ -5954,7 +5985,7 @@ "type": "tidelift" } ], - "time": "2024-08-15T22:48:53+00:00" + "time": "2024-09-20T08:28:38+00:00" }, { "name": "symfony/deprecation-contracts", @@ -6025,16 +6056,16 @@ }, { "name": "symfony/filesystem", - "version": "v7.1.2", + "version": "v7.1.5", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "92a91985250c251de9b947a14bb2c9390b1a562c" + "reference": "61fe0566189bf32e8cfee78335d8776f64a66f5a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/92a91985250c251de9b947a14bb2c9390b1a562c", - "reference": "92a91985250c251de9b947a14bb2c9390b1a562c", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/61fe0566189bf32e8cfee78335d8776f64a66f5a", + "reference": "61fe0566189bf32e8cfee78335d8776f64a66f5a", "shasum": "" }, "require": { @@ -6071,7 +6102,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v7.1.2" + "source": "https://github.com/symfony/filesystem/tree/v7.1.5" }, "funding": [ { @@ -6087,7 +6118,7 @@ "type": "tidelift" } ], - "time": "2024-06-28T10:03:55+00:00" + "time": "2024-09-17T09:16:35+00:00" }, { "name": "symfony/finder", @@ -6536,16 +6567,16 @@ }, { "name": "symfony/process", - "version": "v7.1.3", + "version": "v7.1.5", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "7f2f542c668ad6c313dc4a5e9c3321f733197eca" + "reference": "5c03ee6369281177f07f7c68252a280beccba847" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/7f2f542c668ad6c313dc4a5e9c3321f733197eca", - "reference": "7f2f542c668ad6c313dc4a5e9c3321f733197eca", + "url": "https://api.github.com/repos/symfony/process/zipball/5c03ee6369281177f07f7c68252a280beccba847", + "reference": "5c03ee6369281177f07f7c68252a280beccba847", "shasum": "" }, "require": { @@ -6577,7 +6608,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.1.3" + "source": "https://github.com/symfony/process/tree/v7.1.5" }, "funding": [ { @@ -6593,7 +6624,7 @@ "type": "tidelift" } ], - "time": "2024-07-26T12:44:47+00:00" + "time": "2024-09-19T21:48:23+00:00" }, { "name": "symfony/service-contracts", @@ -6680,16 +6711,16 @@ }, { "name": "symfony/string", - "version": "v7.1.4", + "version": "v7.1.5", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "6cd670a6d968eaeb1c77c2e76091c45c56bc367b" + "reference": "d66f9c343fa894ec2037cc928381df90a7ad4306" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/6cd670a6d968eaeb1c77c2e76091c45c56bc367b", - "reference": "6cd670a6d968eaeb1c77c2e76091c45c56bc367b", + "url": "https://api.github.com/repos/symfony/string/zipball/d66f9c343fa894ec2037cc928381df90a7ad4306", + "reference": "d66f9c343fa894ec2037cc928381df90a7ad4306", "shasum": "" }, "require": { @@ -6747,7 +6778,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v7.1.4" + "source": "https://github.com/symfony/string/tree/v7.1.5" }, "funding": [ { @@ -6763,7 +6794,7 @@ "type": "tidelift" } ], - "time": "2024-08-12T09:59:40+00:00" + "time": "2024-09-20T08:28:38+00:00" }, { "name": "textalk/websocket", @@ -6995,7 +7026,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 5 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index c70d9ca11b..4f8953a270 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -503,7 +503,18 @@ class Deletes extends Action foreach ($collections as $collection) { if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '') || !\in_array($collection->getId(), $projectCollectionIds)) { - $dbForProject->deleteCollection($collection->getId()); + try { + $dbForProject->deleteCollection($collection->getId()); + } catch (Throwable $e) { + Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); + + /** + * Ignore junction tables; + */ + if (!preg_match('/^_\d+_\d+$/', $collection->getId())) { + throw $e; + } + } } else { $this->deleteByGroup($collection->getId(), [], database: $dbForProject); } diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index b7097dbb04..e5d05bb2fb 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Extend\Exception; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; @@ -11,6 +12,10 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; +const METRIC_COLLECTION_LEVEL_STORAGE = 4; +const METRIC_DATABASE_LEVEL_STORAGE = 3; +const METRIC_PROJECT_LEVEL_STORAGE = 2; + class UsageDump extends Action { protected array $stats = []; @@ -70,6 +75,15 @@ class UsageDump extends Action continue; } + if (str_contains($key, METRIC_DATABASES_STORAGE)) { + try { + $this->handleDatabaseStorage($key, $dbForProject); + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); + } + continue; + } + foreach ($this->periods as $period => $format) { $time = 'inf' === $period ? null : date($format, time()); $id = \md5("{$time}_{$period}_{$key}"); @@ -107,4 +121,175 @@ class UsageDump extends Action } } } + + private function handleDatabaseStorage(string $key, Database $dbForProject): void + { + $data = explode('.', $key); + $start = microtime(true); + + $updateMetric = function (Database $dbForProject, int $value, string $key, string $period, string|null $time) { + $id = \md5("{$time}_{$period}_{$key}"); + + try { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + $dbForProject->decreaseDocumentAttribute( + 'stats', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats', + $id, + 'value', + $value + ); + } + } + }; + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + $value = 0; + $diskValue = 0; + $previousValue = 0; + try { + $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); + } catch (\Exception $e) { + // No previous value + } + + switch (count($data)) { + // Collection Level + case METRIC_COLLECTION_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; + $collectionInternalId = $data[1]; + + try { + $value = $dbForProject->getSizeOfCollection('database_'.$databaseInternalId.'_collection_'.$collectionInternalId); + $diskValue = $dbForProject->getSizeOfCollectionOnDisk('database_'.$databaseInternalId.'_collection_'.$collectionInternalId); + } catch (\Exception $e) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + + // Compare with previous value + $diff = $value - $previousValue; + $diskDiff = $diskValue - $previousValue; + + if ($diff === 0 && $diskDiff === 0) { + break; + } + + // Update Collection + $updateMetric($dbForProject, $diff, $key, $period, $time); + $updateMetric($dbForProject, $diskDiff, $key . '_disk', $period, $time); + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $databaseKey . '_disk', $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); + break; + // Database Level + case METRIC_DATABASE_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; + + $collections = []; + try { + $collections = $dbForProject->find('database_' . $databaseInternalId); + } catch (\Exception $e) { + // Database not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + + foreach ($collections as $collection) { + try { + $value += $dbForProject->getSizeOfCollection('database_'.$databaseInternalId.'_collection_'.$collection->getInternalId()); + $diskValue += $dbForProject->getSizeOfCollectionOnDisk('database_'.$databaseInternalId.'_collection_'.$collection->getInternalId()); + } catch (\Exception $e) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + } + + $diff = $value - $previousValue; + $diskDiff = $diskValue - $previousValue; + + if ($diff === 0 && $diskDiff === 0) { + break; + } + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $databaseKey . '_disk', $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); + break; + // Project Level + case METRIC_PROJECT_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); + // Get all project databases + $databases = $dbForProject->find('database'); + + // Recalculate all databases + foreach ($databases as $database) { + $collections = $dbForProject->find('database_' . $database->getInternalId()); + + foreach ($collections as $collection) { + try { + $value += $dbForProject->getSizeOfCollection('database_'.$database->getInternalId().'_collection_'.$collection->getInternalId()); + $diskValue += $dbForProject->getSizeOfCollectionOnDisk('database_'.$database->getInternalId().'_collection_'.$collection->getInternalId()); + } catch (\Exception $e) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + } + } + + $diff = $value - $previousValue; + $diskDiff = $diskValue - $previousValue; + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); + break; + } + } + + $end = microtime(true); + + console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); + } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index a2c07d34b0..6cc2639f51 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -625,6 +625,11 @@ class Response extends SwooleResponse } } + if (!$data->isSet($key) && !$rule['required']) { // set output key null if data key is not set and required is false + $output[$key] = null; + continue; + } + if ($rule['array']) { if (!is_array($data[$key])) { throw new Exception($key . ' must be an array of type ' . $rule['type']); diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php index d4733f2568..eb985baabb 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -28,6 +28,12 @@ class UsageDatabase extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of total storage used in bytes.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('collections', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of collections per period.', @@ -42,6 +48,13 @@ class UsageDatabase extends Model 'example' => [], 'array' => true ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated storage used in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabases.php b/src/Appwrite/Utopia/Response/Model/UsageDatabases.php index f775f9489d..e0abba8ab8 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageDatabases.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabases.php @@ -34,6 +34,12 @@ class UsageDatabases extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of total databases storage in bytes.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('databases', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of databases per period.', @@ -55,6 +61,13 @@ class UsageDatabases extends Model 'example' => [], 'array' => true ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of the aggregated number of databases storage in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 2703691238..17d9271f04 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -28,6 +28,12 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('databasesStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of databases storage size (in bytes).', + 'default' => 0, + 'example' => 0, + ]) ->addRule('usersTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of users.', @@ -118,6 +124,13 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('databasesStorageBreakdown', [ + 'type' => Response::MODEL_METRIC_BREAKDOWN, + 'description' => 'An array of the aggregated breakdown of storage usage by databases.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('executionsMbSecondsBreakdown', [ 'type' => Response::MODEL_METRIC_BREAKDOWN, 'description' => 'Aggregated breakdown in totals of execution mbSeconds by functions.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index d3623acffc..e5b9f0f819 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -143,7 +143,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(20, count($response['body'])); + $this->assertEquals(22, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); @@ -324,7 +324,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(20, count($response['body'])); + $this->assertEquals(22, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -545,7 +545,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(20, count($response['body'])); + $this->assertEquals(22, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); @@ -590,6 +590,260 @@ class UsageTest extends Scope return $data; } + public function testDatabaseStoragePrepare(): array + { + $response = $this->client->call( + Client::METHOD_POST, + '/databases', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'databaseId' => 'unique()', + 'name' => 'dbStorageStats', + ] + ); + + $this->assertNotEmpty($response['body']['$id']); + $databaseId = $response['body']['$id']; + + $response = $this->client->call( + Client::METHOD_POST, + '/databases/' . $databaseId . '/collections', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'collectionId' => 'unique()', + 'name' => 'collectionStorageStats', + 'documentSecurity' => false, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ] + ); + + $this->assertNotEmpty($response['body']['$id']); + $collectionId = $response['body']['$id']; + + $response = $this->client->call( + Client::METHOD_POST, + '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'key' => 'data', + 'size' => 100000, + 'required' => true, + ] + ); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + ]; + } + + // /** @depends testDatabaseStoragePrepare */ + // #[Retry(count: 1)] + // public function testDatabaseStorageStatsCreateDocument(array $data): array + // { + // $databaseId = $data['databaseId']; + // $collectionId = $data['collectionId']; + + // $originalProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $originalProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $originalProjectMetrics['body']); + + // $originalProjectMetrics = $originalProjectMetrics['body']; + + // $originalDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $originalDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $originalDatabaseMetrics['body']); + // $originalDatabaseMetrics = $originalDatabaseMetrics['body']; + + // // Create documents + // for ($i = 0; $i < 100; $i++) { + // $response = $this->client->call( + // Client::METHOD_POST, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', + // array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'documentId' => 'unique()', + // 'data' => ['data' => str_repeat('a', 10000)], + // ] + // ); + + // $this->assertEquals(201, $response['headers']['status-code']); + // } + + // sleep(self::WAIT); + + // for ($i = 0; $i < 3; $i++) { + // try { + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); + // $this->assertGreaterThan($originalProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); + + // $newProjectMetrics = $newProjectMetrics['body']; + + // $newDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); + // $this->assertGreaterThan($originalDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); + + // $newDatabaseMetrics = $newDatabaseMetrics['body']; + + // return [ + // 'databaseId' => $databaseId, + // 'collectionId' => $collectionId, + // 'currentProjectMetrics' => $newProjectMetrics, + // 'currentDatabaseMetrics' => $newDatabaseMetrics, + // ]; + // } catch (ExpectationFailedException $e) { + // if ($i === 2) { + // throw $e; + // } + // sleep(self::WAIT); + // continue; + // } + // } + // } + + // /** @depends testDatabaseStorageStatsCreateDocument */ + // #[Retry(count: 1)] + // public function testDatabaseStorageStatsDeleteDocument(array $data): array + // { + // $databaseId = $data['databaseId']; + // $collectionId = $data['collectionId']; + // $currentProjectMetrics = $data['currentProjectMetrics']; + // $currentDatabaseMetrics = $data['currentDatabaseMetrics']; + + // $documents = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', + // array_merge([ + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'queries' => [ + // Query::limit(50)->toString() + // ] + // ] + // ); + + // foreach ($documents['body']['documents'] as $document) { + // $response = $this->client->call( + // Client::METHOD_DELETE, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document['$id'], + // array_merge([ + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()) + // ); + + // $this->assertEquals(204, $response['headers']['status-code']); + // } + + // sleep(self::WAIT); + + // for ($i = 0; $i < 3; $i++) { + // try { + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); + // $this->assertLessThan($currentProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); + + // $newProjectMetrics = $newProjectMetrics['body']; + + // $newDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); + // $this->assertLessThan($currentDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); + + // $newDatabaseMetrics = $newDatabaseMetrics['body']; + + // return [ + // 'databaseId' => $databaseId, + // 'collectionId' => $collectionId, + // 'currentProjectMetrics' => $newProjectMetrics, + // 'currentDatabaseMetrics' => $newDatabaseMetrics, + // ]; + // } catch (ExpectationFailedException $e) { + // if ($i === 2) { + // throw $e; + // } + // sleep(self::WAIT); + // continue; + // } + // } + + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + // } /** @depends testDatabaseStats */ public function testPrepareFunctionsStats(array $data): array diff --git a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php index ca77cf2581..96bb0b5609 100644 --- a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php +++ b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php @@ -224,7 +224,7 @@ class DatabasesConsoleClientTest extends Scope ]); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(5, count($response['body'])); + $this->assertEquals(7, count($response['body'])); $this->assertEquals('24h', $response['body']['range']); $this->assertIsNumeric($response['body']['documentsTotal']); $this->assertIsNumeric($response['body']['collectionsTotal']); diff --git a/tests/unit/Utopia/ResponseTest.php b/tests/unit/Utopia/ResponseTest.php index cd111ec22c..452119fafb 100644 --- a/tests/unit/Utopia/ResponseTest.php +++ b/tests/unit/Utopia/ResponseTest.php @@ -55,12 +55,32 @@ class ResponseTest extends TestCase 'integer' => 123, 'boolean' => true, 'hidden' => 'secret', + 'array' => [ + 'string 1', + 'string 2' + ], ]), 'single'); $this->assertArrayHasKey('string', $output); $this->assertArrayHasKey('integer', $output); $this->assertArrayHasKey('boolean', $output); $this->assertArrayNotHasKey('hidden', $output); + $this->assertIsArray($output['array']); + + // test optional array + $output = $this->response->output(new Document([ + 'string' => 'lorem ipsum', + 'integer' => 123, + 'boolean' => true, + 'hidden' => 'secret', + ]), 'single'); + $this->assertArrayHasKey('string', $output); + $this->assertArrayHasKey('integer', $output); + $this->assertArrayHasKey('boolean', $output); + $this->assertArrayNotHasKey('hidden', $output); + $this->assertArrayHasKey('array', $output); + $this->assertNull($output['array']); + } public function testResponseModelRequired(): void diff --git a/tests/unit/Utopia/Single.php b/tests/unit/Utopia/Single.php index 3bd09ef6da..b7f36d10a8 100644 --- a/tests/unit/Utopia/Single.php +++ b/tests/unit/Utopia/Single.php @@ -28,6 +28,11 @@ class Single extends Model 'type' => self::TYPE_STRING, 'default' => 'default', 'required' => true + ]) + ->addRule('array', [ + 'type' => self::TYPE_STRING, + 'required' => false, + 'array' => true, ]); }