diff --git a/.env b/.env index 926ada82ba..4d117ce82e 100644 --- a/.env +++ b/.env @@ -38,6 +38,10 @@ _APP_CONNECTIONS_STORAGE=local://localhost _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 +_APP_INFLUXDB_HOST=influxdb +_APP_INFLUXDB_PORT=8086 +_APP_STATSD_HOST=telegraf +_APP_STATSD_PORT=8125 _APP_SMTP_HOST=maildev _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= diff --git a/CHANGES.md b/CHANGES.md index 864fc04012..864784259b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -121,41 +121,6 @@ - Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780) - Fix french translation [#4782](https://github.com/appwrite/appwrite/pull/4782) - Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814) -- New usage metrics collection flow [#4770](https://github.com/appwrite/appwrite/pull/4770) - - Deprecated influxdb, telegraf containers and removed all of their occurrences from the code. - - Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables. - - Removed usage labels dependency. - - Dropped type attribute from stats collection. - - Usage metrics are processed via new usage worker. - - Metrics changes: - - Storage - - deprecated - - filesCreate, filesRead, filesUpdate, filesDelete, bucketsCreate, bucketsRead, bucketsUpdate, bucketsDelete. - - renamed - - filesCount to filesTotal, storage to filesStorage, bucketsCount to bucketsTotal. - - Auth - - deprecated - - usersCreate, usersRead, usersUpdate, usersDelete, sessionsCreate sessionsProviderCreate, sessionsDelete. - - renamed - - usersCount to usersTotal. - - added - - sessionsTotal. - - Databases - - deprecated - - databasesCreate, databasesRead, databasesDelete, documentsCreate, documentsRead, documentsUpdate, documentsDelete, collectionsCreate, collectionsRead, collectionsUpdate, collectionsDelete. - - renamed - - databasesCount to databasesTotal, collectionsCount to collectionsTotal, documentsCount to documentsTotal. - - Functions - - deprecated - - executionsFailure, executionsSuccess, buildsFailure, buildsSuccess, executionsFailure, executionsSuccess. - - renamed - - executionsTime to executionsCompute, buildsTime to buildsCompute, documentsCount to documentsTotal. - - added - - functionsTotal, buildsStorage, deploymentsTotal, deploymentsStorage. - - Project - - renamed - - executions to executionsTotal, builds to buildsTotal, requests to requestsTotal, storage to filesStorage, buckets to bucketsTotal, users to usersTotal, documents to documentsTotal, collections to collectionsTotal, databases to databasesTotal. - ## Bugs - Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83e14d8c31..0cd8574609 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -238,6 +238,8 @@ Appwrite stack is a combination of a variety of open-source technologies and too - Redis - for managing cache and in-memory data (currently, we do not use Redis for persistent data). - MariaDB - for database storage and queries. +- InfluxDB - for managing stats and time-series based data +- Statsd - for sending data over UDP protocol (using Telegraf) - ClamAV - for validating and scanning storage files. - Imagemagick - for manipulating and managing image media files. - Webp - for better compression of images on supporting clients. diff --git a/Dockerfile b/Dockerfile index 7548ec70f4..fb32e2308d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -92,6 +92,10 @@ ENV _APP_SERVER=swoole \ _APP_DB_USER=root \ _APP_DB_PASS=password \ _APP_DB_SCHEMA=appwrite \ + _APP_INFLUXDB_HOST=influxdb \ + _APP_INFLUXDB_PORT=8086 \ + _APP_STATSD_HOST=telegraf \ + _APP_STATSD_PORT=8125 \ _APP_FUNCTIONS_SIZE_LIMIT=30000000 \ _APP_FUNCTIONS_TIMEOUT=900 \ _APP_FUNCTIONS_CONTAINERS=10 \ @@ -161,6 +165,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/patch-delete-project-collections && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ + chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ @@ -180,8 +185,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ - chmod +x /usr/local/bin/worker-migrations && \ - chmod +x /usr/local/bin/worker-usage + chmod +x /usr/local/bin/worker-migrations # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/cli.php b/app/cli.php index d1dd885775..2d12d69adb 100644 --- a/app/cli.php +++ b/app/cli.php @@ -116,6 +116,29 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $getProjectDB; }, ['pools', 'dbForConsole', 'cache']); +CLI::setResource('influxdb', function (Registry $register) { + $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */ + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable $th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); + return $database; +}, ['register']); CLI::setResource('queueForFunctions', function (Group $pools) { return new Func($pools->get('queue')->pop()->getResource()); diff --git a/app/config/collections.php b/app/config/collections.php index 7826d51306..1d452f4b3a 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1302,7 +1302,7 @@ $commonCollections = [ 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 8, - 'signed' => true, + 'signed' => false, 'required' => true, 'default' => null, 'array' => false, @@ -1330,6 +1330,17 @@ $commonCollections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('type'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 1, + 'signed' => false, + 'required' => true, + 'default' => 0, // 0 -> count, 1 -> sum + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -1348,53 +1359,14 @@ $commonCollections = [ ], [ '$id' => ID::custom('_key_metric_period_time'), - 'type' => Database::INDEX_UNIQUE, + 'type' => Database::INDEX_KEY, 'attributes' => ['metric', 'period', 'time'], 'lengths' => [], 'orders' => [Database::ORDER_DESC], ], ], ], - - 'statsLogger' => [ - '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('statsLogger'), - 'name' => 'StatsLogger', - 'attributes' => [ - [ - '$id' => ID::custom('time'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('metrics'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 5012, - 'signed' => true, - 'required' => false, - 'default' => [], - 'array' => false, - 'filters' => ['json'], - ], - ], - 'indexes' => [ - [ - '$id' => ID::custom('_key_time'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['time'], - 'lengths' => [], - 'orders' => [Database::ORDER_DESC], - ], - ], - ], -]; + ]; $projectCollections = array_merge([ 'databases' => [ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 60b635ef3e..7c6cd891ce 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -58,6 +58,7 @@ App::post('/v1/account/invite') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createWithInviteCode') @@ -153,6 +154,7 @@ App::post('/v1/account') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', []) ->label('sdk.namespace', 'account') ->label('sdk.method', 'create') @@ -266,6 +268,8 @@ App::post('/v1/account/sessions/email') ->label('audits.event', 'session.create') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') + ->label('usage.metric', 'sessions.{scope}.requests.create') + ->label('usage.params', ['provider:email']) ->label('sdk.auth', []) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createEmailSession') @@ -518,6 +522,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->label('abuse-limit', 50) ->label('abuse-key', 'ip:{ip}') ->label('docs', false) + ->label('usage.metric', 'sessions.{scope}.requests.create') + ->label('usage.params', ['provider:{request.provider}']) ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.') ->param('code', '', new Text(2048, 0), 'OAuth2 code.', true) ->param('state', '', new Text(2048), 'OAuth2 state params.', true) @@ -1138,6 +1144,8 @@ App::put('/v1/account/sessions/magic-url') ->label('audits.event', 'session.update') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') + ->label('usage.metric', 'sessions.{scope}.requests.create') + ->label('usage.params', ['provider:magic-url']) ->label('sdk.auth', []) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateMagicURLSession') @@ -1512,6 +1520,8 @@ App::post('/v1/account/sessions/anonymous') ->label('audits.event', 'session.create') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') + ->label('usage.metric', 'sessions.{scope}.requests.create') + ->label('usage.params', ['provider:anonymous']) ->label('sdk.auth', []) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createAnonymousSession') @@ -1687,6 +1697,7 @@ App::get('/v1/account') ->desc('Get Account') ->groups(['api', 'account']) ->label('scope', 'account') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'get') @@ -1707,6 +1718,7 @@ App::get('/v1/account/prefs') ->desc('Get Account Preferences') ->groups(['api', 'account']) ->label('scope', 'account') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'getPrefs') @@ -1729,6 +1741,7 @@ App::get('/v1/account/sessions') ->desc('List Sessions') ->groups(['api', 'account']) ->label('scope', 'account') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'listSessions') @@ -1767,6 +1780,7 @@ App::get('/v1/account/logs') ->desc('List Logs') ->groups(['api', 'account']) ->label('scope', 'account') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'listLogs') @@ -1827,6 +1841,7 @@ App::get('/v1/account/sessions/:sessionId') ->desc('Get Session') ->groups(['api', 'account']) ->label('scope', 'account') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'getSession') @@ -1874,6 +1889,7 @@ App::patch('/v1/account/name') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateName') @@ -1908,6 +1924,7 @@ App::patch('/v1/account/password') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePassword') @@ -1973,6 +1990,7 @@ App::patch('/v1/account/email') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateEmail') @@ -2042,6 +2060,7 @@ App::patch('/v1/account/phone') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePhone') @@ -2100,6 +2119,7 @@ App::patch('/v1/account/prefs') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePrefs') @@ -2133,6 +2153,7 @@ App::patch('/v1/account/status') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateStatus') @@ -2176,6 +2197,7 @@ App::delete('/v1/account/sessions/:sessionId') ->label('event', 'users.[userId].sessions.[sessionId].delete') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{user.$id}') + ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'deleteSession') @@ -2252,6 +2274,7 @@ App::patch('/v1/account/sessions/:sessionId') ->label('audits.event', 'session.update') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') + ->label('usage.metric', 'sessions.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateSession') @@ -2336,6 +2359,7 @@ App::delete('/v1/account/sessions') ->label('event', 'users.[userId].sessions.[sessionId].delete') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{user.$id}') + ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'deleteSessions') @@ -2397,6 +2421,7 @@ App::post('/v1/account/recovery') ->label('audits.event', 'recovery.create') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createRecovery') @@ -2537,6 +2562,7 @@ App::put('/v1/account/recovery') ->label('audits.event', 'recovery.update') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateRecovery') @@ -2623,6 +2649,7 @@ App::post('/v1/account/verification') ->label('event', 'users.[userId].verification.[tokenId].create') ->label('audits.event', 'verification.create') ->label('audits.resource', 'user/{response.userId}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createVerification') @@ -2742,6 +2769,7 @@ App::put('/v1/account/verification') ->label('event', 'users.[userId].verification.[tokenId].update') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{response.userId}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateVerification') @@ -2802,6 +2830,7 @@ App::post('/v1/account/verification/phone') ->label('event', 'users.[userId].verification.[tokenId].create') ->label('audits.event', 'verification.create') ->label('audits.resource', 'user/{response.userId}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createPhoneVerification') @@ -2897,6 +2926,7 @@ App::put('/v1/account/verification/phone') ->label('event', 'users.[userId].verification.[tokenId].update') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{response.userId}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePhoneVerification') diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index aa7999b844..8c857b2fc4 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -376,6 +376,7 @@ App::post('/v1/databases') ->label('scope', 'databases.write') ->label('audits.event', 'database.create') ->label('audits.resource', 'database/{response.$id}') + ->label('usage.metric', 'databases.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'create') @@ -449,6 +450,7 @@ App::get('/v1/databases') ->desc('List Databases') ->groups(['api', 'database']) ->label('scope', 'databases.read') + ->label('usage.metric', 'databases.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'list') @@ -494,6 +496,7 @@ App::get('/v1/databases/:databaseId') ->desc('Get Database') ->groups(['api', 'database']) ->label('scope', 'databases.read') + ->label('usage.metric', 'databases.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'get') @@ -608,6 +611,7 @@ App::put('/v1/databases/:databaseId') ->label('event', 'databases.[databaseId].update') ->label('audits.event', 'database.update') ->label('audits.resource', 'database/{response.$id}') + ->label('usage.metric', 'databases.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'update') @@ -652,6 +656,7 @@ App::delete('/v1/databases/:databaseId') ->label('event', 'databases.[databaseId].delete') ->label('audits.event', 'database.delete') ->label('audits.resource', 'database/{request.databaseId}') + ->label('usage.metric', 'databases.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'delete') @@ -697,6 +702,8 @@ App::post('/v1/databases/:databaseId/collections') ->label('scope', 'collections.write') ->label('audits.event', 'collection.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}') + ->label('usage.metric', 'collections.{scope}.requests.create') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'createCollection') @@ -762,6 +769,8 @@ App::get('/v1/databases/:databaseId/collections') ->desc('List Collections') ->groups(['api', 'database']) ->label('scope', 'collections.read') + ->label('usage.metric', 'collections.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listCollections') @@ -817,6 +826,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId') ->desc('Get Collection') ->groups(['api', 'database']) ->label('scope', 'collections.read') + ->label('usage.metric', 'collections.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getCollection') @@ -851,6 +862,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') ->desc('List Collection Logs') ->groups(['api', 'database']) ->label('scope', 'collections.read') + ->label('usage.metric', 'collections.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listCollectionLogs') @@ -948,6 +961,8 @@ App::put('/v1/databases/:databaseId/collections/:collectionId') ->label('event', 'databases.[databaseId].collections.[collectionId].update') ->label('audits.event', 'collection.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateCollection') @@ -1016,6 +1031,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') ->label('event', 'databases.[databaseId].collections.[collectionId].delete') ->label('audits.event', 'collection.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.delete') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'deleteCollection') @@ -1070,6 +1087,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'createStringAttribute') @@ -1126,6 +1145,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email' ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createEmailAttribute') @@ -1168,6 +1189,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createEnumAttribute') @@ -1226,6 +1249,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip') ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createIpAttribute') @@ -1268,6 +1293,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url') ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createUrlAttribute') @@ -1310,6 +1337,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createIntegerAttribute') @@ -1381,6 +1410,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float' ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createFloatAttribute') @@ -1455,6 +1486,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createBooleanAttribute') @@ -1496,6 +1529,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createDatetimeAttribute') @@ -1540,6 +1575,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createRelationshipAttribute') @@ -1617,6 +1654,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes') ->desc('List Attributes') ->groups(['api', 'database']) ->label('scope', 'collections.read') + ->label('usage.metric', 'collections.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listAttributes') @@ -1678,6 +1717,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key') ->desc('Get Attribute') ->groups(['api', 'database']) ->label('scope', 'collections.read') + ->label('usage.metric', 'collections.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getAttribute') @@ -1755,6 +1796,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateStringAttribute') @@ -1794,6 +1837,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/email ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateEmailAttribute') @@ -1833,6 +1878,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/ ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateEnumAttribute') @@ -1874,6 +1921,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/ip/:k ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateIpAttribute') @@ -1913,6 +1962,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/url/: ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateUrlAttribute') @@ -1952,6 +2003,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateIntegerAttribute') @@ -2001,6 +2054,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateFloatAttribute') @@ -2050,6 +2105,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/boole ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateBooleanAttribute') @@ -2088,6 +2145,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/datet ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'documents.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateDatetimeAttribute') @@ -2126,6 +2185,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/ ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'documents.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateRelationshipAttribute') @@ -2180,6 +2241,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete') ->label('audits.event', 'attribute.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'deleteAttribute') @@ -2289,6 +2352,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') ->label('scope', 'collections.write') ->label('audits.event', 'index.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'createIndex') @@ -2444,6 +2509,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes') ->desc('List Indexes') ->groups(['api', 'database']) ->label('scope', 'collections.read') + ->label('usage.metric', 'collections.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listIndexes') @@ -2505,6 +2572,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') ->desc('Get Index') ->groups(['api', 'database']) ->label('scope', 'collections.read') + ->label('usage.metric', 'collections.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getIndex') @@ -2547,6 +2616,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete') ->label('audits.event', 'index.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'collections.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'deleteIndex') @@ -2610,7 +2681,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].create') ->label('scope', 'documents.write') ->label('audits.event', 'document.create') - ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('usage.metric', 'documents.{scope}.requests.create') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -2846,6 +2919,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->desc('List Documents') ->groups(['api', 'database']) ->label('scope', 'documents.read') + ->label('usage.metric', 'documents.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listDocuments') @@ -2976,6 +3051,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->desc('Get Document') ->groups(['api', 'database']) ->label('scope', 'documents.read') + ->label('usage.metric', 'documents.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getDocument') @@ -3070,6 +3147,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->desc('List Document Logs') ->groups(['api', 'database']) ->label('scope', 'documents.read') + ->label('usage.metric', 'documents.{scope}.requests.read') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listDocumentLogs') @@ -3172,6 +3251,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->label('scope', 'documents.write') ->label('audits.event', 'document.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') + ->label('usage.metric', 'documents.{scope}.requests.update') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -3401,6 +3482,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete') ->label('audits.event', 'document.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}') + ->label('usage.metric', 'documents.{scope}.requests.delete') + ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -3559,7 +3642,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu App::get('/v1/databases/usage') ->desc('Get usage stats for the database') - ->groups(['api', 'database', 'usage']) + ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'databases') @@ -3572,62 +3655,112 @@ App::get('/v1/databases/usage') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - METRIC_DATABASES, - METRIC_COLLECTIONS, - METRIC_DOCUMENTS, - ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], ]; + + $metrics = [ + 'databases.$all.count.total', + 'documents.$all.count.total', + 'collections.$all.count.total', + 'databases.$all.requests.create', + 'databases.$all.requests.read', + 'databases.$all.requests.update', + 'databases.$all.requests.delete', + 'collections.$all.requests.create', + 'collections.$all.requests.read', + 'collections.$all.requests.update', + 'collections.$all.requests.delete', + 'documents.$all.requests.create', + 'documents.$all.requests.read', + 'documents.$all.requests.update', + 'documents.$all.requests.delete' + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + // Added 3'rd level to Index [period, metric, time] because of order by. + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'databasesCount' => $stats['databases.$all.count.total'] ?? [], + 'documentsCount' => $stats['documents.$all.count.total'] ?? [], + 'collectionsCount' => $stats['collections.$all.count.total'] ?? [], + 'documentsCreate' => $stats['documents.$all.requests.create'] ?? [], + 'documentsRead' => $stats['documents.$all.requests.read'] ?? [], + 'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [], + 'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [], + 'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [], + 'collectionsRead' => $stats['collections.$all.requests.read'] ?? [], + 'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [], + 'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [], + 'databasesCreate' => $stats['databases.$all.requests.create'] ?? [], + 'databasesRead' => $stats['databases.$all.requests.read'] ?? [], + 'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [], + 'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [], + ]); } - } - $response->dynamic(new Document([ - 'range' => $range, - 'databasesTotal' => $usage[$metrics[0]], - 'collectionsTotal' => $usage[$metrics[1]], - 'documentsTotal' => $usage[$metrics[2]], - ]), Response::MODEL_USAGE_DATABASES); + + $response->dynamic($usage, Response::MODEL_USAGE_DATABASES); }); App::get('/v1/databases/:databaseId/usage') ->desc('Get usage stats for the database') - ->groups(['api', 'database', 'usage']) + ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'databases') @@ -3641,68 +3774,103 @@ App::get('/v1/databases/:databaseId/usage') ->inject('dbForProject') ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) { - $database = $dbForProject->getDocument('databases', $databaseId); - - if ($database->isEmpty()) { - throw new Exception(Exception::DATABASE_NOT_FOUND); - } - - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS), - str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS), - ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], ]; - } - } - $response->dynamic(new Document([ - 'range' => $range, - 'collectionsTotal' => $usage[$metrics[0]], - 'documentsTotal' => $usage[$metrics[1]], - ]), Response::MODEL_USAGE_DATABASE); + $metrics = [ + 'collections.' . $databaseId . '.count.total', + 'collections.' . $databaseId . '.requests.create', + 'collections.' . $databaseId . '.requests.read', + 'collections.' . $databaseId . '.requests.update', + 'collections.' . $databaseId . '.requests.delete', + 'documents.' . $databaseId . '.count.total', + 'documents.' . $databaseId . '.requests.create', + 'documents.' . $databaseId . '.requests.read', + 'documents.' . $databaseId . '.requests.update', + 'documents.' . $databaseId . '.requests.delete' + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + // TODO@kodumbeats explore performance if query is ordered by time ASC + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [], + 'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [], + 'collectionsRead' => $stats["collections.{$databaseId}.requests.read"] ?? [], + 'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [], + 'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [], + 'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [], + 'documentsCreate' => $stats["documents.{$databaseId}.requests.create"] ?? [], + 'documentsRead' => $stats["documents.{$databaseId}.requests.read"] ?? [], + 'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [], + 'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [], + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_DATABASE); }); App::get('/v1/databases/:databaseId/collections/:collectionId/usage') ->alias('/v1/database/:collectionId/usage', ['databaseId' => 'default']) ->desc('Get usage stats for a collection') - ->groups(['api', 'database', 'usage']) + ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'databases') @@ -3725,52 +3893,84 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), - ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], ]; - } - } - $response->dynamic(new Document([ - 'range' => $range, - 'documentsTotal' => $usage[$metrics[0]], - ]), Response::MODEL_USAGE_COLLECTION); + $metrics = [ + "documents.{$databaseId}/{$collectionId}.count.total", + "documents.{$databaseId}/{$collectionId}.requests.create", + "documents.{$databaseId}/{$collectionId}.requests.read", + "documents.{$databaseId}/{$collectionId}.requests.update", + "documents.{$databaseId}/{$collectionId}.requests.delete", + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [], + 'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [], + 'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [], + 'documentsUpdate' => $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [], + 'documentsDelete' => $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION); }); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fcc9d1fea2..daa5d3e0a1 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -451,66 +451,92 @@ App::get('/v1/functions/:functionId/usage') throw new Exception(Exception::FUNCTION_NOT_FOUND); } - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS), - str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), - str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), - str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), - str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), - str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), - str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), - ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], ]; - } - } - $response->dynamic(new Document([ - 'range' => $range, - 'deploymentsTotal' => $usage[$metrics[0]], - 'deploymentsStorage' => $usage[$metrics[1]], - 'buildsTotal' => $usage[$metrics[2]], - 'buildsStorage' => $usage[$metrics[3]], - 'buildsTime' => $usage[$metrics[4]], - 'executionsTotal' => $usage[$metrics[5]], - 'executionsTime' => $usage[$metrics[6]], - ]), Response::MODEL_USAGE_FUNCTION); + $metrics = [ + "executions.$functionId.compute.total", + "executions.$functionId.compute.success", + "executions.$functionId.compute.failure", + "executions.$functionId.compute.time", + "builds.$functionId.compute.total", + "builds.$functionId.compute.success", + "builds.$functionId.compute.failure", + "builds.$functionId.compute.time", + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [], + 'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [], + 'executionsSuccesse' => $stats["executions.$functionId.compute.success"] ?? [], + 'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [], + 'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [], + 'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [], + 'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [], + 'buildsTime' => $stats["builds.$functionId.compute.time" ?? []] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_FUNCTION); }); App::get('/v1/functions/usage') @@ -528,67 +554,92 @@ App::get('/v1/functions/usage') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - METRIC_FUNCTIONS, - METRIC_DEPLOYMENTS, - METRIC_DEPLOYMENTS_STORAGE, - METRIC_BUILDS, - METRIC_BUILDS_STORAGE, - METRIC_BUILDS_COMPUTE, - METRIC_EXECUTIONS, - METRIC_EXECUTIONS_COMPUTE, - ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], ]; + + $metrics = [ + 'executions.$all.compute.total', + 'executions.$all.compute.failure', + 'executions.$all.compute.success', + 'executions.$all.compute.time', + 'builds.$all.compute.total', + 'builds.$all.compute.failure', + 'builds.$all.compute.success', + 'builds.$all.compute.time', + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'executionsTotal' => $stats[$metrics[0]] ?? [], + 'executionsFailure' => $stats[$metrics[1]] ?? [], + 'executionsSuccess' => $stats[$metrics[2]] ?? [], + 'executionsTime' => $stats[$metrics[3]] ?? [], + 'buildsTotal' => $stats[$metrics[4]] ?? [], + 'buildsFailure' => $stats[$metrics[5]] ?? [], + 'buildsSuccess' => $stats[$metrics[6]] ?? [], + 'buildsTime' => $stats[$metrics[7]] ?? [], + ]); } - } - $response->dynamic(new Document([ - 'range' => $range, - 'functionsTotal' => $usage[$metrics[0]], - 'deploymentsTotal' => $usage[$metrics[1]], - 'deploymentsStorage' => $usage[$metrics[2]], - 'buildsTotal' => $usage[$metrics[3]], - 'buildsStorage' => $usage[$metrics[4]], - 'buildsTime' => $usage[$metrics[5]], - 'executionsTotal' => $usage[$metrics[6]], - 'executionsTime' => $usage[$metrics[7]], - ]), Response::MODEL_USAGE_FUNCTIONS); + + $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS); }); App::put('/v1/functions/:functionId') @@ -1438,11 +1489,11 @@ App::post('/v1/functions/:functionId/executions') ->inject('dbForProject') ->inject('user') ->inject('events') + ->inject('usage') ->inject('mode') ->inject('queueForFunctions') ->inject('geodb') - ->inject('queueForUsage') - ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, string $mode, Func $queueForFunctions, Reader $geodb, Usage $queueForUsage) { + ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1645,13 +1696,6 @@ App::post('/v1/functions/:functionId/executions') $execution->setAttribute('logs', $executionResponse['logs']); $execution->setAttribute('errors', $executionResponse['errors']); $execution->setAttribute('duration', $executionResponse['duration']); - /** - * Sync execution compute usage from - */ - $queueForUsage - ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000)) // per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000)) // per function - ; } catch (\Throwable $th) { $durationEnd = \microtime(true); @@ -1668,6 +1712,14 @@ App::post('/v1/functions/:functionId/executions') $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); } + // TODO revise this later using route label + $usage + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')); // ms + + $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 18217b6f34..1adf4f735a 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -3,7 +3,6 @@ use Appwrite\Extend\Exception; use Appwrite\Utopia\Response; use Utopia\App; -use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate as DuplicateException; @@ -15,10 +14,11 @@ use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; +use Utopia\Database\DateTime; App::get('/v1/project/usage') ->desc('Get usage stats for a project') - ->groups(['api', 'usage']) + ->groups(['api']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'project') @@ -30,76 +30,94 @@ App::get('/v1/project/usage') ->inject('response') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - METRIC_NETWORK_REQUESTS, - METRIC_NETWORK_INBOUND, - METRIC_NETWORK_OUTBOUND, - METRIC_EXECUTIONS, - METRIC_DOCUMENTS, - METRIC_DATABASES, - METRIC_USERS, - METRIC_BUCKETS, - METRIC_FILES_STORAGE - ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], ]; + + $metrics = [ + 'project.$all.network.requests', + 'project.$all.network.bandwidth', + 'project.$all.storage.size', + 'users.$all.count.total', + 'databases.$all.count.total', + 'documents.$all.count.total', + 'executions.$all.compute.total', + 'buckets.$all.count.total' + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'requests' => $stats[$metrics[0]] ?? [], + 'network' => $stats[$metrics[1]] ?? [], + 'storage' => $stats[$metrics[2]] ?? [], + 'users' => $stats[$metrics[3]] ?? [], + 'databases' => $stats[$metrics[4]] ?? [], + 'documents' => $stats[$metrics[5]] ?? [], + 'executions' => $stats[$metrics[6]] ?? [], + 'buckets' => $stats[$metrics[7]] ?? [], + ]); } - } - - $response->dynamic(new Document([ - 'range' => $range, - 'requestsTotal' => ($usage[$metrics[0]]), - 'network' => ($usage[$metrics[1]] + $usage[$metrics[2]]), - 'executionsTotal' => $usage[$metrics[3]], - 'documentsTotal' => $usage[$metrics[4]], - 'databasesTotal' => $usage[$metrics[5]], - 'usersTotal' => $usage[$metrics[6]], - 'bucketsTotal' => $usage[$metrics[7]], - 'filesStorage' => $usage[$metrics[8]], - ]), Response::MODEL_USAGE_PROJECT); + $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); }); - // Variables App::post('/v1/project/variables') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index da488b062b..7901c587fb 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -95,6 +95,7 @@ App::post('/v1/projects') $backups['database_db_fra1_03'] = ['from' => '10:30', 'to' => '11:15']; $backups['database_db_fra1_04'] = ['from' => '13:30', 'to' => '14:15']; $backups['database_db_fra1_05'] = ['from' => '4:30', 'to' => '5:15']; + $backups['database_db_fra1_06'] = ['from' => '16:30', 'to' => '17:15']; $databases = Config::getParam('pools-database', []); @@ -118,7 +119,7 @@ App::post('/v1/projects') } } - if ($index = array_search('database_db_fra1_05', $databases)) { + if ($index = array_search('database_db_fra1_06', $databases)) { $database = $databases[$index]; } else { $database = $databases[array_rand($databases)]; @@ -286,6 +287,120 @@ App::get('/v1/projects/:projectId') $response->dynamic($project, Response::MODEL_PROJECT); }); +App::get('/v1/projects/:projectId/usage') + ->desc('Get usage stats for a project') + ->groups(['api', 'projects', 'usage']) + ->label('scope', 'projects.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'projects') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT) + ->param('projectId', '', new UID(), 'Project unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForConsole') + ->inject('dbForProject') + ->inject('register') + ->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) { + + $project = $dbForConsole->getDocument('projects', $projectId); + + if ($project->isEmpty()) { + throw new Exception(Exception::PROJECT_NOT_FOUND); + } + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $dbForProject->setNamespace("_{$project->getInternalId()}"); + + $metrics = [ + 'project.$all.network.requests', + 'project.$all.network.bandwidth', + 'project.$all.storage.size', + 'users.$all.count.total', + 'databases.$all.count.total', + 'documents.$all.count.total', + 'executions.$all.compute.total', + 'buckets.$all.count.total' + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'requests' => $stats[$metrics[0]] ?? [], + 'network' => $stats[$metrics[1]] ?? [], + 'storage' => $stats[$metrics[2]] ?? [], + 'users' => $stats[$metrics[3]] ?? [], + 'databases' => $stats[$metrics[4]] ?? [], + 'documents' => $stats[$metrics[5]] ?? [], + 'executions' => $stats[$metrics[6]] ?? [], + 'buckets' => $stats[$metrics[7]] ?? [], + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); + }); + App::patch('/v1/projects/:projectId') ->desc('Update Project') ->groups(['api', 'projects']) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index c34515b5e1..7d39e1c137 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -51,6 +51,7 @@ App::post('/v1/storage/buckets') ->label('event', 'buckets.[bucketId].create') ->label('audits.event', 'bucket.create') ->label('audits.resource', 'bucket/{response.$id}') + ->label('usage.metric', 'buckets.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'createBucket') @@ -146,6 +147,7 @@ App::get('/v1/storage/buckets') ->desc('List buckets') ->groups(['api', 'storage']) ->label('scope', 'buckets.read') + ->label('usage.metric', 'buckets.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'listBuckets') @@ -192,6 +194,7 @@ App::get('/v1/storage/buckets/:bucketId') ->desc('Get Bucket') ->groups(['api', 'storage']) ->label('scope', 'buckets.read') + ->label('usage.metric', 'buckets.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getBucket') @@ -220,6 +223,7 @@ App::put('/v1/storage/buckets/:bucketId') ->label('event', 'buckets.[bucketId].update') ->label('audits.event', 'bucket.update') ->label('audits.resource', 'bucket/{response.$id}') + ->label('usage.metric', 'buckets.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'updateBucket') @@ -287,6 +291,7 @@ App::delete('/v1/storage/buckets/:bucketId') ->label('audits.event', 'bucket.delete') ->label('event', 'buckets.[bucketId].delete') ->label('audits.resource', 'bucket/{request.bucketId}') + ->label('usage.metric', 'buckets.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'deleteBucket') @@ -329,6 +334,8 @@ App::post('/v1/storage/buckets/:bucketId/files') ->label('audits.event', 'file.create') ->label('event', 'buckets.[bucketId].files.[fileId].create') ->label('audits.resource', 'file/{response.$id}') + ->label('usage.metric', 'files.{scope}.requests.create') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -669,6 +676,8 @@ App::get('/v1/storage/buckets/:bucketId/files') ->groups(['api', 'storage']) ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) + ->label('usage.metric', 'files.{scope}.requests.read') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'listFiles') ->label('sdk.description', '/docs/references/storage/list-files.md') @@ -744,6 +753,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId') ->groups(['api', 'storage']) ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) + ->label('usage.metric', 'files.{scope}.requests.read') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFile') ->label('sdk.description', '/docs/references/storage/get-file.md') @@ -791,6 +802,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->label('cache', true) ->label('cache.resourceType', 'bucket/{request.bucketId}') ->label('cache.resource', 'file/{request.fileId}') + ->label('usage.metric', 'files.{scope}.requests.read') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFilePreview') @@ -952,6 +965,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->desc('Get File for Download') ->groups(['api', 'storage']) ->label('scope', 'files.read') + ->label('usage.metric', 'files.{scope}.requests.read') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFileDownload') @@ -1090,6 +1105,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->desc('Get File for View') ->groups(['api', 'storage']) ->label('scope', 'files.read') + ->label('usage.metric', 'files.{scope}.requests.read') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFileView') @@ -1242,6 +1259,8 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') ->label('event', 'buckets.[bucketId].files.[fileId].update') ->label('audits.event', 'file.update') ->label('audits.resource', 'file/{response.$id}') + ->label('usage.metric', 'files.{scope}.requests.update') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -1348,6 +1367,8 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->label('event', 'buckets.[bucketId].files.[fileId].delete') ->label('audits.event', 'file.delete') ->label('audits.resource', 'file/{request.fileId}') + ->label('usage.metric', 'files.{scope}.requests.delete') + ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -1449,62 +1470,103 @@ App::get('/v1/storage/usage') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - METRIC_BUCKETS, - METRIC_FILES, - METRIC_FILES_STORAGE, - ]; + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; + $metrics = [ + 'project.$all.storage.size', + 'buckets.$all.count.total', + 'buckets.$all.requests.create', + 'buckets.$all.requests.read', + 'buckets.$all.requests.update', + 'buckets.$all.requests.delete', + 'files.$all.storage.size', + 'files.$all.count.total', + 'files.$all.requests.create', + 'files.$all.requests.read', + 'files.$all.requests.update', + 'files.$all.requests.delete', + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); } - } - }); + }); - $format = (match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }); - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, - ]; - } + $usage = new Document([ + 'range' => $range, + 'bucketsCount' => $stats['buckets.$all.count.total'], + 'bucketsCreate' => $stats['buckets.$all.requests.create'], + 'bucketsRead' => $stats['buckets.$all.requests.read'], + 'bucketsUpdate' => $stats['buckets.$all.requests.update'], + 'bucketsDelete' => $stats['buckets.$all.requests.delete'], + 'storage' => $stats['project.$all.storage.size'], + 'filesCount' => $stats['files.$all.count.total'], + 'filesCreate' => $stats['files.$all.requests.create'], + 'filesRead' => $stats['files.$all.requests.read'], + 'filesUpdate' => $stats['files.$all.requests.update'], + 'filesDelete' => $stats['files.$all.requests.delete'], + ]); } - $response->dynamic(new Document([ - 'range' => $range, - 'bucketsTotal' => $usage[$metrics[0]], - 'filesTotal' => $usage[$metrics[1]], - 'filesStorage' => $usage[$metrics[2]], - ]), Response::MODEL_USAGE_STORAGE); + $response->dynamic($usage, Response::MODEL_USAGE_STORAGE); }); App::get('/v1/storage/:bucketId/usage') - ->desc('Get usage stats for storage bucket') + ->desc('Get usage stats for a storage bucket') ->groups(['api', 'storage', 'usage']) ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) @@ -1525,55 +1587,86 @@ App::get('/v1/storage/:bucketId/usage') throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), - str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), - ]; + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; + $metrics = [ + "files.{$bucketId}.count.total", + "files.{$bucketId}.storage.size", + "files.{$bucketId}.requests.create", + "files.{$bucketId}.requests.read", + "files.{$bucketId}.requests.update", + "files.{$bucketId}.requests.delete", + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); } - } - }); + }); - $format = (match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }); - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, - ]; - } + $usage = new Document([ + 'range' => $range, + 'filesCount' => $stats[$metrics[0]], + 'filesStorage' => $stats[$metrics[1]], + 'filesCreate' => $stats[$metrics[2]], + 'filesRead' => $stats[$metrics[3]], + 'filesUpdate' => $stats[$metrics[4]], + 'filesDelete' => $stats[$metrics[5]], + ]); } - $response->dynamic(new Document([ - 'range' => $range, - 'filesTotal' => $usage[$metrics[0]], - 'filesStorage' => $usage[$metrics[1]], - ]), Response::MODEL_USAGE_BUCKETS); + $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 3969ace459..1ea7e099fb 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -114,6 +114,7 @@ App::post('/v1/users') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'create') @@ -146,6 +147,7 @@ App::post('/v1/users/bcrypt') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createBcryptUser') @@ -176,6 +178,7 @@ App::post('/v1/users/md5') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createMD5User') @@ -206,6 +209,7 @@ App::post('/v1/users/argon2') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createArgon2User') @@ -236,6 +240,7 @@ App::post('/v1/users/sha') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createSHAUser') @@ -273,6 +278,7 @@ App::post('/v1/users/phpass') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createPHPassUser') @@ -303,6 +309,7 @@ App::post('/v1/users/scrypt') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createScryptUser') @@ -346,6 +353,7 @@ App::post('/v1/users/scrypt-modified') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createScryptModifiedUser') @@ -376,6 +384,7 @@ App::get('/v1/users') ->desc('List Users') ->groups(['api', 'users']) ->label('scope', 'users.read') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'list') @@ -422,6 +431,7 @@ App::get('/v1/users/:userId') ->desc('Get User') ->groups(['api', 'users']) ->label('scope', 'users.read') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'get') @@ -447,6 +457,7 @@ App::get('/v1/users/:userId/prefs') ->desc('Get User Preferences') ->groups(['api', 'users']) ->label('scope', 'users.read') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'getPrefs') @@ -474,6 +485,7 @@ App::get('/v1/users/:userId/sessions') ->desc('List User Sessions') ->groups(['api', 'users']) ->label('scope', 'users.read') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listSessions') @@ -515,6 +527,7 @@ App::get('/v1/users/:userId/memberships') ->desc('List User Memberships') ->groups(['api', 'users']) ->label('scope', 'users.read') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listMemberships') @@ -554,6 +567,7 @@ App::get('/v1/users/:userId/logs') ->desc('List User Logs') ->groups(['api', 'users']) ->label('scope', 'users.read') + ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listLogs') @@ -686,6 +700,7 @@ App::patch('/v1/users/:userId/status') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateStatus') @@ -721,7 +736,6 @@ App::put('/v1/users/:userId/labels') ->label('scope', 'users.write') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('audits.userId', '{response.$id}') ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') @@ -760,6 +774,7 @@ App::patch('/v1/users/:userId/verification/phone') ->label('scope', 'users.write') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePhoneVerification') @@ -796,6 +811,7 @@ App::patch('/v1/users/:userId/name') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateName') @@ -833,6 +849,7 @@ App::patch('/v1/users/:userId/password') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePassword') @@ -897,6 +914,7 @@ App::patch('/v1/users/:userId/email') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateEmail') @@ -952,6 +970,7 @@ App::patch('/v1/users/:userId/phone') ->label('scope', 'users.write') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePhone') @@ -996,6 +1015,7 @@ App::patch('/v1/users/:userId/verification') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{request.userId}') ->label('audits.userId', '{request.userId}') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateEmailVerification') @@ -1028,6 +1048,7 @@ App::patch('/v1/users/:userId/prefs') ->groups(['api', 'users']) ->label('event', 'users.[userId].update.prefs') ->label('scope', 'users.write') + ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePrefs') @@ -1063,6 +1084,7 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->label('scope', 'users.write') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{request.userId}') + ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteSession') @@ -1106,6 +1128,7 @@ App::delete('/v1/users/:userId/sessions') ->label('scope', 'users.write') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{user.$id}') + ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteSessions') @@ -1148,6 +1171,7 @@ App::delete('/v1/users/:userId') ->label('scope', 'users.write') ->label('audits.event', 'user.delete') ->label('audits.resource', 'user/{request.userId}') + ->label('usage.metric', 'users.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'delete') @@ -1226,59 +1250,96 @@ App::get('/v1/users/usage') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true) ->inject('response') ->inject('dbForProject') ->inject('register') - ->action(function (string $range, Response $response, Database $dbForProject) { + ->action(function (string $range, string $provider, Response $response, Database $dbForProject) { - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - METRIC_USERS, - METRIC_SESSIONS, - ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric] = []; - foreach ($results as $result) { - $stats[$metric][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric][] = [ - 'value' => $stats[$metric][$formatDate]['value'] ?? 0, - 'date' => $formatDate, + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], ]; - } - } - $response->dynamic(new Document([ - 'range' => $range, - 'usersTotal' => $usage[$metrics[0]], - 'sessionsTotal' => $usage[$metrics[1]], - ]), Response::MODEL_USAGE_USERS); + $metrics = [ + 'users.$all.count.total', + 'users.$all.requests.create', + 'users.$all.requests.read', + 'users.$all.requests.update', + 'users.$all.requests.delete', + 'sessions.$all.requests.create', + 'sessions.$all.requests.delete', + "sessions.$provider.requests.create", + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match ($period) { // convert period to seconds for unix timestamp math + '1h' => 3600, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), + ]; + $backfill--; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'usersCount' => $stats['users.$all.count.total'] ?? [], + 'usersCreate' => $stats['users.$all.requests.create'] ?? [], + 'usersRead' => $stats['users.$all.requests.read'] ?? [], + 'usersUpdate' => $stats['users.$all.requests.update'] ?? [], + 'usersDelete' => $stats['users.$all.requests.delete'] ?? [], + 'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [], + 'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [], + 'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_USERS); }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index f9fc0cba98..aafd862c3f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -8,8 +8,8 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Extend\Exception; -use Appwrite\Event\Usage; use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Usage\Stats; use Appwrite\Utopia\Response; use Appwrite\Utopia\Request; use Utopia\App; @@ -48,99 +48,43 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar return $label; }; -$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) { - - $value = 1; +$databaseListener = function (string $event, Document $document, Stats $usage) { + $multiplier = 1; if ($event === Database::EVENT_DOCUMENT_DELETE) { - $value = -1; + $multiplier = -1; } - switch (true) { - case $document->getCollection() === 'teams': - $queueForUsage - ->addMetric(METRIC_TEAMS, $value); // per project + $collection = $document->getCollection(); + switch ($collection) { + case 'users': + $usage->setParam('users.{scope}.count.total', 1 * $multiplier); break; - case $document->getCollection() === 'users': - $queueForUsage - ->addMetric(METRIC_USERS, $value); // per project - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } + case 'databases': + $usage->setParam('databases.{scope}.count.total', 1 * $multiplier); break; - case $document->getCollection() === 'sessions': // sessions - $queueForUsage - ->addMetric(METRIC_SESSIONS, $value); //per project + case 'buckets': + $usage->setParam('buckets.{scope}.count.total', 1 * $multiplier); break; - case $document->getCollection() === 'databases': // databases - $queueForUsage - ->addMetric(METRIC_DATABASES, $value); // per project - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $queueForUsage - ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database - ; - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $collectionInternalId = $parts[3] ?? 0; - $queueForUsage - ->addMetric(METRIC_DOCUMENTS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection - break; - case $document->getCollection() === 'buckets': //buckets - $queueForUsage - ->addMetric(METRIC_BUCKETS, $value); // per project - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'bucket_'): // files - $queueForUsage - ->addMetric(METRIC_FILES, $value) // per project - ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project - ->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES), $value) // per bucket - ->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket - break; - case $document->getCollection() === 'functions': - $queueForUsage - ->addMetric(METRIC_FUNCTIONS, $value); // per project - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case $document->getCollection() === 'deployments': - $queueForUsage - ->addMetric(METRIC_DEPLOYMENTS, $value) // per project - ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);// per function - - break; - case $document->getCollection() === 'executions': - $queueForUsage - ->addMetric(METRIC_EXECUTIONS, $value) // per project - ->addMetric(str_replace('{functionInternalId}', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function + case 'deployments': + $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier); break; default: + if (strpos($collection, 'bucket_') === 0) { + $usage + ->setParam('bucketId', $document->getAttribute('bucketId')) + ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier) + ->setParam('files.{scope}.count.total', 1 * $multiplier); + } elseif (strpos($collection, 'database_') === 0) { + $usage + ->setParam('databaseId', $document->getAttribute('databaseId')); + if (strpos($collection, '_collection_') !== false) { + $usage + ->setParam('collectionId', $document->getAttribute('$collectionId')) + ->setParam('documents.{scope}.count.total', 1 * $multiplier); + } else { + $usage->setParam('collections.{scope}.count.total', 1 * $multiplier); + } + } break; } }; @@ -157,10 +101,10 @@ App::init() ->inject('deletes') ->inject('database') ->inject('dbForProject') - ->inject('queueForUsage') ->inject('mode') ->inject('mails') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Usage $queueForUsage, string $mode, Mail $mails) use ($databaseListener) { + ->inject('usage') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode, Mail $mails, Stats $usage) use ($databaseListener) { $route = $utopia->getRoute(); @@ -242,25 +186,19 @@ App::init() ->setProject($project) ->setUser($user); - $smtp = $project->getAttribute('smtp', []); - if (!empty($smtp) && ($smtp['enabled'] ?? false)) { - $mails - ->setSmtpHost($smtp['host'] ?? '') - ->setSmtpPort($smtp['port'] ?? 25) - ->setSmtpUsername($smtp['username'] ?? '') - ->setSmtpPassword($smtp['password'] ?? '') - ->setSmtpSenderEmail($smtp['sender'] ?? '') - ->setSmtpReplyTo($smtp['replyTo'] ?? ''); - } + $usage + ->setParam('projectInternalId', $project->getInternalId()) + ->setParam('projectId', $project->getId()) + ->setParam('project.{scope}.network.requests', 1) + ->setParam('httpMethod', $request->getMethod()) + ->setParam('project.{scope}.network.inbound', 0) + ->setParam('project.{scope}.network.outbound', 0); $deletes->setProject($project); $database->setProject($project); - $calculateUsage = fn ($event, Document $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject); - - $dbForProject - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', $calculateUsage) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', $calculateUsage); + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage)); + $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage)); $useCache = $route->getLabel('cache', false); @@ -423,14 +361,14 @@ App::shutdown() ->inject('user') ->inject('events') ->inject('audits') + ->inject('usage') ->inject('deletes') ->inject('database') ->inject('dbForProject') ->inject('queueForFunctions') - ->inject('queueForUsage') ->inject('mode') ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, string $mode, Database $dbForConsole) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -583,46 +521,35 @@ App::shutdown() } } - if ($project->getId() !== 'console') { - if ($mode !== APP_MODE_ADMIN) { - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } + if ( + App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled' + && $project->getId() + && !empty($route->getLabel('sdk.namespace', null)) + ) { // Don't calculate console usage on admin mode + $metric = $route->getLabel('usage.metric', ''); + $usageParams = $route->getLabel('usage.params', []); - $queueForUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); - } - - $queueForUsage - ->setProject($project) - ->trigger(); - } - - /** - * Update user last activity - */ - if (!$user->isEmpty()) { - $accessedAt = $user->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCCESS)) > $accessedAt) { - $user->setAttribute('accessedAt', DateTime::now()); - - if (APP_MODE_ADMIN !== $mode) { - $dbForProject->updateDocument('users', $user->getId(), $user); - } else { - $dbForConsole->updateDocument('users', $user->getId(), $user); + if (!empty($metric)) { + $usage->setParam($metric, 1); + foreach ($usageParams as $param) { + $param = $parseLabel($param, $responsePayload, $requestParams, $user); + $parts = explode(':', $param); + if (count($parts) != 2) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set'); + } + $usage->setParam($parts[0], $parts[1]); } } - } - }); -App::init() - ->groups(['usage']) - ->action(function () { - if (App::getEnv('_APP_USAGE_STATS', 'enabled') !== 'enabled') { - throw new Exception(Exception::GENERAL_USAGE_DISABLED); + $fileSize = 0; + $file = $request->getFiles('file'); + if (!empty($file)) { + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + } + + $usage + ->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize) + ->setParam('project.{scope}.network.outbound', $response->getSize()) + ->submit(); } }); diff --git a/app/init.php b/app/init.php index 3d0e652b6b..d5d0ea9748 100644 --- a/app/init.php +++ b/app/init.php @@ -32,6 +32,7 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\Origin; use Appwrite\OpenSSL\OpenSSL; use Appwrite\URL\URL as AppwriteURL; +use Appwrite\Usage\Stats; use Utopia\App; use Utopia\Logger\Logger; use Utopia\Cache\Adapter\Redis as RedisCache; @@ -236,7 +237,6 @@ Config::load('providers', __DIR__ . '/config/providers.php'); Config::load('platforms', __DIR__ . '/config/platforms.php'); Config::load('collections', __DIR__ . '/config/collections.php'); Config::load('runtimes', __DIR__ . '/config/runtimes.php'); -Config::load('usage', __DIR__ . '/config/usage.php'); Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes Config::load('services', __DIR__ . '/config/services.php'); // List of services @@ -770,6 +770,31 @@ $register->set('pools', function () { return $group; }); +$register->set('influxdb', function () { + + // Register DB connection + $host = App::getEnv('_APP_INFLUXDB_HOST', ''); + $port = App::getEnv('_APP_INFLUXDB_PORT', ''); + + if (empty($host) || empty($port)) { + return; + } + $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}"); + $client = new InfluxDB\Client($host, $port, '', '', false, false, 5); + $client->setDriver($driver); + + return $client; +}); +$register->set('statsd', function () { + // Register DB connection + $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); + $port = App::getEnv('_APP_STATSD_PORT', 8125); + + $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port); + $statsd = new \Domnikl\Statsd\Client($connection); + + return $statsd; +}); $register->set('smtp', function () { $mail = new PHPMailer(true); @@ -873,9 +898,9 @@ App::setResource('queue', function (Group $pools) { App::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); -App::setResource('queueForUsage', function (Connection $queue) { - return new Usage($queue); -}, ['queue']); +App::setResource('usage', function ($register) { + return new Stats($register->get('statsd')); +}, ['register']); App::setResource('clients', function ($request, $console, $project) { $console->setAttribute('platforms', [ // Always allow current host '$collection' => ID::custom('platforms'), diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 43d993ace6..4d8f4510f8 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -80,6 +80,7 @@ services: - mariadb - redis # - clamav + - influxdb environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -115,6 +116,8 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_USAGE_STATS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - _APP_STORAGE_PREVIEW_LIMIT - _APP_STORAGE_ANTIVIRUS @@ -131,6 +134,8 @@ services: - _APP_EXECUTOR_HOST - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + - _APP_STATSD_HOST + - _APP_STATSD_PORT - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -468,6 +473,35 @@ services: - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY + appwrite-usage: + image: /: + entrypoint: usage + container_name: appwrite-usage + <<: *x-logging + restart: unless-stopped + networks: + - appwrite + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + appwrite-schedule: image: /: entrypoint: schedule @@ -552,41 +586,26 @@ services: # volumes: # - appwrite-uploads:/storage/uploads - appwrite-worker-usage: - image: /: - entrypoint: worker-usage + influxdb: + image: appwrite/influxdb:1.5.0 + container_name: appwrite-influxdb <<: *x-logging - container_name: appwrite-worker-usage + restart: unless-stopped networks: - appwrite volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb + - appwrite-influxdb:/var/lib/influxdb:rw + + telegraf: + image: appwrite/telegraf:1.4.0 + container_name: appwrite-telegraf + <<: *x-logging + restart: unless-stopped + networks: + - appwrite environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_CONNECTIONS_MAX - - _APP_POOL_CLIENTS - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_CONNECTIONS_DB_CONSOLE - - _APP_CONNECTIONS_DB_PROJECT - - _APP_CONNECTIONS_CACHE - - _APP_CONNECTIONS_QUEUE - - _APP_USAGE_STATS - - _APP_LOGGING_PROVIDER - - _APP_LOGGING_CONFIG + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT networks: gateway: @@ -602,6 +621,7 @@ volumes: appwrite-cache: appwrite-uploads: appwrite-certificates: + appwrite-influxdb: appwrite-config: appwrite-functions: appwrite-builds: diff --git a/app/worker.php b/app/worker.php index ea086fa43d..75fc95e9ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -4,10 +4,12 @@ require_once __DIR__ . '/init.php'; use Appwrite\Event\Func; use Appwrite\Event\Usage; +use Appwrite\Usage\Stats; use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; +use Utopia\CLI\CLI; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; @@ -86,22 +88,16 @@ Server::setResource('queueForFunctions', function (Registry $register) { ); }, ['register']); -Server::setResource('queueForUsage', function (Registry $register) { - $pools = $register->get('pools'); - return new Usage( - $pools - ->get('queue') - ->pop() - ->getResource() - ); -}, ['register']); - Server::setResource('log', fn() => new Log()); Server::setResource('logger', function ($register) { return $register->get('logger'); }, ['register']); +Server::setResource('statsd', function ($register) { + return $register->get('statsd'); +}, ['register']); + Server::setResource('pools', function ($register) { return $register->get('pools'); }, ['register']); diff --git a/app/workers/builds.php b/app/workers/builds.php index f25a2ec7ce..faf2165d81 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -478,7 +478,7 @@ class BuildsV1 extends Worker * Send realtime Event */ $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $build, project: $project @@ -490,19 +490,23 @@ class BuildsV1 extends Worker channels: $target['channels'], roles: $target['roles'] ); - } - /** Trigger usage queue */ - $this - ->getUsageQueue() - ->setProject($project) - ->addMetric(METRIC_BUILDS, 1) // per project - ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->trigger(); + /** Update usage stats */ + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $statsd = $register->get('statsd'); + $usage = new Stats($statsd); + $usage + ->setParam('projectInternalId', $project->getInternalId()) + ->setParam('projectId', $project->getId()) + ->setParam('functionId', $function->getId()) + ->setParam('builds.{scope}.compute', 1) + ->setParam('buildStatus', $build->getAttribute('status', '')) + ->setParam('buildTime', $build->getAttribute('duration')) + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit(); + } + } } protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForConsole): void diff --git a/app/workers/functions.php b/app/workers/functions.php index 7d0c55da0c..cae3a73cc8 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,18 +2,18 @@ require_once __DIR__ . '/../worker.php'; -use Appwrite\Event\Usage; +use Domnikl\Statsd\Client; use Utopia\Queue\Message; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Usage\Stats; use Appwrite\Utopia\Response\Model\Execution; use Executor\Executor; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -31,7 +31,7 @@ Server::setResource('execute', function () { Log $log, Func $queueForFunctions, Database $dbForProject, - Usage $queueForUsage, + Client $statsd, Document $project, Document $function, string $trigger, @@ -47,7 +47,6 @@ Server::setResource('execute', function () { ) { $user ??= new Document(); $functionId = $function->getId(); - $functionInternalId = $function->getInternalId(); $deploymentId = $function->getAttribute('deployment', ''); $log->addTag('functionId', $functionId); @@ -55,7 +54,6 @@ Server::setResource('execute', function () { /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); - $deploymentInternalId = $deployment->getInternalId(); if ($deployment->getAttribute('resourceId') !== $functionId) { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); @@ -129,14 +127,6 @@ Server::setResource('execute', function () { if ($execution->isEmpty()) { throw new Exception('Failed to create or read execution'); } - - /** - * Usage - */ - - $queueForUsage - ->addMetric(METRIC_EXECUTIONS, 1) // per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1); // per function } if ($execution->getAttribute('status') !== 'processing') { @@ -186,13 +176,13 @@ Server::setResource('execute', function () { $executionResponse = $executor->createExecution( projectId: $project->getId(), deploymentId: $deploymentId, - version: $function->getAttribute('version'), body: \strlen($body) > 0 ? $body : null, variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), + version: $function->getAttribute('version'), path: $path, method: $method, headers: $headers, @@ -275,13 +265,24 @@ Server::setResource('execute', function () { roles: $target['roles'] ); - /** Trigger usage queue */ - $queueForUsage - ->setProject($project) - ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) - ->trigger() - ; + /** Update usage stats */ + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $usage = new Stats($statsd); + $usage + ->setParam('projectId', $project->getId()) + ->setParam('projectInternalId', $project->getInternalId()) + ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')) + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit(); + } + + if (!empty($error)) { + throw new Exception($error, $errorCode); + } }; }); @@ -289,10 +290,10 @@ $server->job() ->inject('message') ->inject('dbForProject') ->inject('queueForFunctions') - ->inject('queueForUsage') + ->inject('statsd') ->inject('execute') ->inject('log') - ->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, callable $execute, Log $log) { + ->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute, Log $log) { $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -334,7 +335,7 @@ $server->job() } Console::success('Iterating function: ' . $function->getAttribute('name')); $execute( - queueForUsage: $queueForUsage, + statsd: $statsd, dbForProject: $dbForProject, project: $project, function: $function, @@ -382,7 +383,7 @@ $server->job() path: $payload['path'], method: $payload['method'], headers: $payload['headers'], - queueForUsage: $queueForUsage, + statsd: $statsd, ); break; case 'schedule': @@ -402,7 +403,7 @@ $server->job() path: $payload['path'], method: $payload['method'], headers: $payload['headers'], - queueForUsage: $queueForUsage, + statsd: $statsd, ); break; } diff --git a/app/workers/usage.php b/app/workers/usage.php deleted file mode 100644 index 0c9dcbc5aa..0000000000 --- a/app/workers/usage.php +++ /dev/null @@ -1,288 +0,0 @@ -deployments>build || documents>collection>database || buckets>files. - * When we remove a parent document we need to deduct his children aggregation from the project scope. - */ -Server::setResource('reduce', function (Cache $cache, Registry $register, $pools) { - return function ($database, $projectInternalId, Document $document, array &$metrics) use ($pools, $cache, $register): void { - try { - $dbForProject = new Database( - $pools - ->get($database) - ->pop() - ->getResource(), - $cache - ); - - $dbForProject->setNamespace('_' . $projectInternalId); - - switch (true) { - case $document->getCollection() === 'users': // users - $sessions = count($document->getAttribute(METRIC_SESSIONS, 0)); - if (!empty($sessions)) { - $metrics[] = [ - 'key' => METRIC_SESSIONS, - 'value' => ($sessions * -1), - ]; - } - break; - case $document->getCollection() === 'databases': // databases - $collections = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS))); - $documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS))); - if (!empty($collections['value'])) { - $metrics[] = [ - 'key' => METRIC_COLLECTIONS, - 'value' => ($collections['value'] * -1), - ]; - } - - if (!empty($documents['value'])) { - $metrics[] = [ - 'key' => METRIC_DOCUMENTS, - 'value' => ($documents['value'] * -1), - ]; - } - break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS))); - - if (!empty($documents['value'])) { - $metrics[] = [ - 'key' => METRIC_DOCUMENTS, - 'value' => ($documents['value'] * -1), - ]; - $metrics[] = [ - 'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), - 'value' => ($documents['value'] * -1), - ]; - } - break; - - case $document->getCollection() === 'buckets': - $files = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES))); - $storage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE))); - - if (!empty($files['value'])) { - $metrics[] = [ - 'key' => METRIC_FILES, - 'value' => ($files['value'] * -1), - ]; - } - - if (!empty($storage['value'])) { - $metrics[] = [ - 'key' => METRIC_FILES_STORAGE, - 'value' => ($storage['value'] * -1), - ]; - } - break; - - case $document->getCollection() === 'functions': - $deployments = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); - $deploymentsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); - $builds = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); - $buildsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); - $buildsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); - $executions = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); - $executionsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); - - if (!empty($deployments['value'])) { - $metrics[] = [ - 'key' => METRIC_DEPLOYMENTS, - 'value' => ($deployments['value'] * -1), - ]; - } - - if (!empty($deploymentsStorage['value'])) { - $metrics[] = [ - 'key' => METRIC_DEPLOYMENTS_STORAGE, - 'value' => ($deploymentsStorage['value'] * -1), - ]; - } - - if (!empty($builds['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS, - 'value' => ($builds['value'] * -1), - ]; - } - - if (!empty($buildsStorage['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_STORAGE, - 'value' => ($buildsStorage['value'] * -1), - ]; - } - - if (!empty($buildsCompute['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE, - 'value' => ($buildsCompute['value'] * -1), - ]; - } - - if (!empty($executions['value'])) { - $metrics[] = [ - 'key' => METRIC_EXECUTIONS, - 'value' => ($executions['value'] * -1), - ]; - } - - if (!empty($executionsCompute['value'])) { - $metrics[] = [ - 'key' => METRIC_EXECUTIONS_COMPUTE, - 'value' => ($executionsCompute['value'] * -1), - ]; - } - break; - default: - break; - } - } catch (\Exception $e) { - console::error("[reducer] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}"); - } finally { - $pools->reclaim(); - } - }; -}, ['cache', 'register', 'pools']); - - -$server->job() - ->inject('message') - ->inject('reduce') - - ->action(function (Message $message, callable $reduce) use (&$stats) { - - $payload = $message->getPayload() ?? []; - $project = new Document($payload['project'] ?? []); - $projectId = $project->getInternalId(); - foreach ($payload['reduce'] ?? [] as $document) { - if (empty($document)) { - continue; - } - - $reduce( - database: $project->getAttribute('database'), - projectInternalId: $project->getInternalId(), - document: new Document($document), - metrics: $payload['metrics'], - ); - } - - $stats[$projectId]['database'] = $project->getAttribute('database'); - foreach ($payload['metrics'] ?? [] as $metric) { - if (!isset($stats[$projectId]['keys'][$metric['key']])) { - $stats[$projectId]['keys'][$metric['key']] = $metric['value']; - continue; - } - $stats[$projectId]['keys'][$metric['key']] += $metric['value']; - } - }); - -$server - ->workerStart() - ->inject('register') - ->inject('cache') - ->inject('pools') - ->action(function ($register, $cache, $pools) use ($periods, &$stats) { - Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) { - - $offset = count($stats); - $projects = array_slice($stats, 0, $offset, true); - array_splice($stats, 0, $offset); - - foreach ($projects as $projectInternalId => $project) { - try { - $dbForProject = new Database( - $pools - ->get($project['database']) - ->pop() - ->getResource(), - $cache - ); - - $dbForProject->setNamespace('_' . $projectInternalId); - - foreach ($project['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - foreach ($periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => App::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } - } - } - if (!empty($project['keys'])) { - $dbForProject->createDocument('statsLogger', new Document([ - 'time' => DateTime::now(), - 'metrics' => $project['keys'], - ])); - } - } catch (\Exception $e) { - $now = DateTime::now(); - console::error("[Error] " . " Time: {$now} " . " projectInternalId: {$projectInternalId}" . " File: {$e->getFile()}" . " Line: {$e->getLine()} " . " message: {$e->getMessage()}"); - } finally { - $pools->reclaim(); - } - } - }); - }); -$server->start(); diff --git a/bin/usage b/bin/usage new file mode 100644 index 0000000000..2709200ae4 --- /dev/null +++ b/bin/usage @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage $@ \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index 9d325ac46e..0000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -QUEUE=v1-usage php /usr/src/code/app/workers/usage.php $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 96d5a27f92..b8c7e6b477 100644 --- a/composer.json +++ b/composer.json @@ -70,6 +70,7 @@ "resque/php-resque": "1.3.6", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", + "influxdb/influxdb-php": "1.15.2", "phpmailer/phpmailer": "6.8.0", "chillerlan/php-qrcode": "4.3.4", "adhocore/jwt": "1.1.2", diff --git a/composer.lock b/composer.lock index 669e1d7f43..e86397caec 100644 --- a/composer.lock +++ b/composer.lock @@ -520,6 +520,398 @@ ], "time": "2022-09-10T18:51:20+00:00" }, + { + "name": "guzzlehttp/guzzle", + "version": "7.7.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/guzzle.git", + "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/fb7566caccf22d74d1ab270de3551f72a58399f5", + "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5", + "shasum": "" + }, + "require": { + "ext-json": "*", + "guzzlehttp/promises": "^1.5.3 || ^2.0", + "guzzlehttp/psr7": "^1.9.1 || ^2.4.5", + "php": "^7.2.5 || ^8.0", + "psr/http-client": "^1.0", + "symfony/deprecation-contracts": "^2.2 || ^3.0" + }, + "provide": { + "psr/http-client-implementation": "1.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.1", + "ext-curl": "*", + "php-http/client-integration-tests": "dev-master#2c025848417c1135031fdf9c728ee53d0a7ceaee as 3.0.999", + "php-http/message-factory": "^1.1", + "phpunit/phpunit": "^8.5.29 || ^9.5.23", + "psr/log": "^1.1 || ^2.0 || ^3.0" + }, + "suggest": { + "ext-curl": "Required for CURL handler support", + "ext-intl": "Required for Internationalized Domain Name (IDN) support", + "psr/log": "Required for using the Log middleware" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "GuzzleHttp\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Jeremy Lindblom", + "email": "jeremeamia@gmail.com", + "homepage": "https://github.com/jeremeamia" + }, + { + "name": "George Mponos", + "email": "gmponos@gmail.com", + "homepage": "https://github.com/gmponos" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://github.com/sagikazarmark" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + } + ], + "description": "Guzzle is a PHP HTTP client library", + "keywords": [ + "client", + "curl", + "framework", + "http", + "http client", + "psr-18", + "psr-7", + "rest", + "web service" + ], + "support": { + "issues": "https://github.com/guzzle/guzzle/issues", + "source": "https://github.com/guzzle/guzzle/tree/7.7.0" + }, + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://github.com/Nyholm", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/guzzlehttp/guzzle", + "type": "tidelift" + } + ], + "time": "2023-05-21T14:04:53+00:00" + }, + { + "name": "guzzlehttp/promises", + "version": "2.0.1", + "source": { + "type": "git", + "url": "https://github.com/guzzle/promises.git", + "reference": "111166291a0f8130081195ac4556a5587d7f1b5d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/promises/zipball/111166291a0f8130081195ac4556a5587d7f1b5d", + "reference": "111166291a0f8130081195ac4556a5587d7f1b5d", + "shasum": "" + }, + "require": { + "php": "^7.2.5 || ^8.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.1", + "phpunit/phpunit": "^8.5.29 || ^9.5.23" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Promise\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + } + ], + "description": "Guzzle promises library", + "keywords": [ + "promise" + ], + "support": { + "issues": "https://github.com/guzzle/promises/issues", + "source": "https://github.com/guzzle/promises/tree/2.0.1" + }, + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://github.com/Nyholm", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/guzzlehttp/promises", + "type": "tidelift" + } + ], + "time": "2023-08-03T15:11:55+00:00" + }, + { + "name": "guzzlehttp/psr7", + "version": "2.6.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/psr7.git", + "reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/8bd7c33a0734ae1c5d074360512beb716bef3f77", + "reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77", + "shasum": "" + }, + "require": { + "php": "^7.2.5 || ^8.0", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.1 || ^2.0", + "ralouphie/getallheaders": "^3.0" + }, + "provide": { + "psr/http-factory-implementation": "1.0", + "psr/http-message-implementation": "1.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.1", + "http-interop/http-factory-tests": "^0.9", + "phpunit/phpunit": "^8.5.29 || ^9.5.23" + }, + "suggest": { + "laminas/laminas-httphandlerrunner": "Emit PSR-7 responses" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Psr7\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "George Mponos", + "email": "gmponos@gmail.com", + "homepage": "https://github.com/gmponos" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://github.com/sagikazarmark" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://sagikazarmark.hu" + } + ], + "description": "PSR-7 message implementation that also provides common utility methods", + "keywords": [ + "http", + "message", + "psr-7", + "request", + "response", + "stream", + "uri", + "url" + ], + "support": { + "issues": "https://github.com/guzzle/psr7/issues", + "source": "https://github.com/guzzle/psr7/tree/2.6.0" + }, + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://github.com/Nyholm", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/guzzlehttp/psr7", + "type": "tidelift" + } + ], + "time": "2023-08-03T15:06:02+00:00" + }, + { + "name": "influxdb/influxdb-php", + "version": "1.15.2", + "source": { + "type": "git", + "url": "https://github.com/influxdata/influxdb-php.git", + "reference": "d6e59f4f04ab9107574fda69c2cbe36671253d03" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/influxdata/influxdb-php/zipball/d6e59f4f04ab9107574fda69c2cbe36671253d03", + "reference": "d6e59f4f04ab9107574fda69c2cbe36671253d03", + "shasum": "" + }, + "require": { + "guzzlehttp/guzzle": "^6.0|^7.0", + "php": "^5.5 || ^7.0 || ^8.0" + }, + "require-dev": { + "dms/phpunit-arraysubset-asserts": "^0.2.1", + "phpunit/phpunit": "^9.5" + }, + "suggest": { + "ext-curl": "Curl extension, needed for Curl driver", + "stefanotorresi/influxdb-php-async": "An asyncronous client for InfluxDB, implemented via ReactPHP." + }, + "type": "library", + "autoload": { + "psr-4": { + "InfluxDB\\": "src/InfluxDB" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Stephen Hoogendijk", + "email": "stephen@tca0.nl" + }, + { + "name": "Daniel Martinez", + "email": "danimartcas@hotmail.com" + }, + { + "name": "Gianluca Arbezzano", + "email": "gianarb92@gmail.com" + } + ], + "description": "InfluxDB client library for PHP", + "keywords": [ + "client", + "influxdata", + "influxdb", + "influxdb class", + "influxdb client", + "influxdb library", + "time series" + ], + "support": { + "issues": "https://github.com/influxdata/influxdb-php/issues", + "source": "https://github.com/influxdata/influxdb-php/tree/1.15.2" + }, + "abandoned": true, + "time": "2020-12-26T17:45:17+00:00" + }, { "name": "jean85/pretty-package-versions", "version": "1.6.0", @@ -992,6 +1384,166 @@ ], "time": "2023-03-06T14:43:22+00:00" }, + { + "name": "psr/http-client", + "version": "1.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-client.git", + "reference": "0955afe48220520692d2d09f7ab7e0f93ffd6a31" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-client/zipball/0955afe48220520692d2d09f7ab7e0f93ffd6a31", + "reference": "0955afe48220520692d2d09f7ab7e0f93ffd6a31", + "shasum": "" + }, + "require": { + "php": "^7.0 || ^8.0", + "psr/http-message": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Client\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP clients", + "homepage": "https://github.com/php-fig/http-client", + "keywords": [ + "http", + "http-client", + "psr", + "psr-18" + ], + "support": { + "source": "https://github.com/php-fig/http-client/tree/1.0.2" + }, + "time": "2023-04-10T20:12:12+00:00" + }, + { + "name": "psr/http-factory", + "version": "1.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-factory.git", + "reference": "e616d01114759c4c489f93b099585439f795fe35" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-factory/zipball/e616d01114759c4c489f93b099585439f795fe35", + "reference": "e616d01114759c4c489f93b099585439f795fe35", + "shasum": "" + }, + "require": { + "php": ">=7.0.0", + "psr/http-message": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interfaces for PSR-7 HTTP message factories", + "keywords": [ + "factory", + "http", + "message", + "psr", + "psr-17", + "psr-7", + "request", + "response" + ], + "support": { + "source": "https://github.com/php-fig/http-factory/tree/1.0.2" + }, + "time": "2023-04-10T20:10:41+00:00" + }, + { + "name": "psr/http-message", + "version": "2.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-message.git", + "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-message/zipball/402d35bcb92c70c026d1a6a9883f06b2ead23d71", + "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP messages", + "homepage": "https://github.com/php-fig/http-message", + "keywords": [ + "http", + "http-message", + "psr", + "psr-7", + "request", + "response" + ], + "support": { + "source": "https://github.com/php-fig/http-message/tree/2.0" + }, + "time": "2023-04-04T09:54:51+00:00" + }, { "name": "psr/log", "version": "1.1.4", @@ -1042,6 +1594,50 @@ }, "time": "2021-05-03T11:20:27+00:00" }, + { + "name": "ralouphie/getallheaders", + "version": "3.0.3", + "source": { + "type": "git", + "url": "https://github.com/ralouphie/getallheaders.git", + "reference": "120b605dfeb996808c31b6477290a714d356e822" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ralouphie/getallheaders/zipball/120b605dfeb996808c31b6477290a714d356e822", + "reference": "120b605dfeb996808c31b6477290a714d356e822", + "shasum": "" + }, + "require": { + "php": ">=5.6" + }, + "require-dev": { + "php-coveralls/php-coveralls": "^2.1", + "phpunit/phpunit": "^5 || ^6.5" + }, + "type": "library", + "autoload": { + "files": [ + "src/getallheaders.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ralph Khattar", + "email": "ralph.khattar@gmail.com" + } + ], + "description": "A polyfill for getallheaders.", + "support": { + "issues": "https://github.com/ralouphie/getallheaders/issues", + "source": "https://github.com/ralouphie/getallheaders/tree/develop" + }, + "time": "2019-03-08T08:55:37+00:00" + }, { "name": "resque/php-resque", "version": "v1.3.6", @@ -1181,6 +1777,73 @@ }, "time": "2021-06-04T20:33:46+00:00" }, + { + "name": "symfony/deprecation-contracts", + "version": "v3.3.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/deprecation-contracts.git", + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/7c3aff79d10325257a001fcf92d991f24fc967cf", + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.4-dev" + }, + "thanks": { + "name": "symfony/contracts", + "url": "https://github.com/symfony/contracts" + } + }, + "autoload": { + "files": [ + "function.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "A generic function and convention to trigger deprecation notices", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.3.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-05-23T14:45:45+00:00" + }, { "name": "symfony/polyfill-php80", "version": "v1.27.0", @@ -5441,5 +6104,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index f55c0d48c0..57f12f96e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,6 +138,8 @@ services: - _APP_SMTP_PASSWORD - _APP_USERS_STATS_RECIPIENTS - _APP_USAGE_STATS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - _APP_STORAGE_PREVIEW_LIMIT - _APP_STORAGE_ANTIVIRUS @@ -154,6 +156,8 @@ services: - _APP_EXECUTOR_HOST - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + - _APP_STATSD_HOST + - _APP_STATSD_PORT - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -689,18 +693,19 @@ services: - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_SCHEDULES - appwrite-worker-usage: - entrypoint: worker-usage + appwrite-usage: + entrypoint: usage <<: *x-logging - container_name: appwrite-worker-usage + container_name: appwrite-usage image: appwrite-dev networks: - appwrite volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - ./dev:/usr/local/dev depends_on: - - redis + - influxdb - mariadb environment: - _APP_ENV @@ -713,6 +718,9 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -869,6 +877,26 @@ services: # - appwrite # volumes: # - appwrite-uploads:/storage/uploads + + influxdb: + image: appwrite/influxdb:1.5.0 + container_name: appwrite-influxdb + <<: *x-logging + networks: + - appwrite + volumes: + - appwrite-influxdb:/var/lib/influxdb:rw + + telegraf: + image: appwrite/telegraf:1.4.0 + container_name: appwrite-telegraf + <<: *x-logging + networks: + - appwrite + environment: + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + # Dev Tools Start ------------------------------------------------------------------------------------------ # # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack @@ -879,6 +907,7 @@ services: # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks # RedisCommander - A nice UI for exploring Redis data # Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks + # Chronograf - A nice UI for exploring InfluxDB data # Webgrind - A nice UI for exploring and debugging code-level stuff maildev: # used mainly for dev tests @@ -941,6 +970,25 @@ services: # - RESQUE_WEB_PORT=6379 # - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user # - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password + + # chronograf: + # image: chronograf:1.6 + # container_name: appwrite-chronograf + # networks: + # - appwrite + # volumes: + # - appwrite-chronograf:/var/lib/chronograf + # ports: + # - "8888:8888" + # environment: + # - INFLUXDB_URL=http://influxdb:8086 + # - KAPACITOR_URL=http://kapacitor:9092 + # - AUTH_DURATION=48h + # - TOKEN_SECRET=duperduper5674829!jwt + # - GH_CLIENT_ID=d86f7145a41eacfc52cc + # - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab + # - GH_ORGS=appwrite + # webgrind: # image: 'jokkedk/webgrind:latest' # volumes: @@ -976,6 +1024,8 @@ volumes: appwrite-cache: appwrite-uploads: appwrite-certificates: + appwrite-influxdb: + appwrite-config: appwrite-functions: appwrite-builds: - appwrite-config: + # appwrite-chronograf: diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 9e919127bf..10c573da42 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -30,6 +30,7 @@ class Tasks extends Service $this->type = self::TYPE_CLI; $this ->addAction(Version::getName(), new Version()) + ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) ->addAction(Hamster::getName(), new Hamster()) diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php new file mode 100644 index 0000000000..fa677ea142 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -0,0 +1,60 @@ +desc('Schedules syncing data from influxdb to Appwrite console db') + ->inject('dbForConsole') + ->inject('influxdb') + ->inject('register') + ->inject('getProjectDB') + ->inject('logError') + ->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError)); + } + + protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void + { + } + + public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError) + { + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action); + + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + $region = App::getEnv('region', 'default'); + $usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger); + + Console::loop(function () use ($interval, $usage) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds"); + $loopStart = microtime(true); + + $usage->collect(); + + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); + } +} diff --git a/src/Appwrite/Usage/Calculator.php b/src/Appwrite/Usage/Calculator.php new file mode 100644 index 0000000000..37c130a34a --- /dev/null +++ b/src/Appwrite/Usage/Calculator.php @@ -0,0 +1,15 @@ +region = $region; + } + + abstract public function collect(): void; +} diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php new file mode 100644 index 0000000000..e0a12b443f --- /dev/null +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -0,0 +1,557 @@ + '1h', + 'startTime' => '-24 hours' + ], + [ + 'key' => '1d', + 'startTime' => '-30 days' + ] + ]; + + /** + * All the metrics that we are collecting + * + * @var array + */ + protected array $metrics = [ + 'project.$all.network.requests' => [ + 'table' => 'appwrite_usage_project_{scope}_network_requests', + ], + 'project.$all.network.bandwidth' => [ + 'table' => 'appwrite_usage_project_{scope}_network_bandwidth', + ], + 'project.$all.network.inbound' => [ + 'table' => 'appwrite_usage_project_{scope}_network_inbound', + ], + 'project.$all.network.outbound' => [ + 'table' => 'appwrite_usage_project_{scope}_network_outbound', + ], + /* Users service metrics */ + 'users.$all.requests.create' => [ + 'table' => 'appwrite_usage_users_{scope}_requests_create', + ], + 'users.$all.requests.read' => [ + 'table' => 'appwrite_usage_users_{scope}_requests_read', + ], + 'users.$all.requests.update' => [ + 'table' => 'appwrite_usage_users_{scope}_requests_update', + ], + 'users.$all.requests.delete' => [ + 'table' => 'appwrite_usage_users_{scope}_requests_delete', + ], + + 'databases.$all.requests.create' => [ + 'table' => 'appwrite_usage_databases_{scope}_requests_create', + ], + 'databases.$all.requests.read' => [ + 'table' => 'appwrite_usage_databases_{scope}_requests_read', + ], + 'databases.$all.requests.update' => [ + 'table' => 'appwrite_usage_databases_{scope}_requests_update', + ], + 'databases.$all.requests.delete' => [ + 'table' => 'appwrite_usage_databases_{scope}_requests_delete', + ], + + 'collections.$all.requests.create' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_create', + ], + 'collections.$all.requests.read' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_read', + ], + 'collections.$all.requests.update' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_update', + ], + 'collections.$all.requests.delete' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_delete', + ], + + 'documents.$all.requests.create' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_create', + ], + 'documents.$all.requests.read' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_read', + ], + 'documents.$all.requests.update' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_update', + ], + 'documents.$all.requests.delete' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_delete', + ], + + 'collections.databaseId.requests.create' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_create', + 'groupBy' => ['databaseId'], + ], + 'collections.databaseId.requests.read' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_read', + 'groupBy' => ['databaseId'], + ], + 'collections.databaseId.requests.update' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_update', + 'groupBy' => ['databaseId'], + ], + 'collections.databaseId.requests.delete' => [ + 'table' => 'appwrite_usage_collections_{scope}_requests_delete', + 'groupBy' => ['databaseId'], + ], + + 'documents.databaseId.requests.create' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_create', + 'groupBy' => ['databaseId'], + ], + 'documents.databaseId.requests.read' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_read', + 'groupBy' => ['databaseId'], + ], + 'documents.databaseId.requests.update' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_update', + 'groupBy' => ['databaseId'], + ], + 'documents.databaseId.requests.delete' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_delete', + 'groupBy' => ['databaseId'], + ], + + 'documents.databaseId/collectionId.requests.create' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_create', + 'groupBy' => ['databaseId', 'collectionId'], + ], + 'documents.databaseId/collectionId.requests.read' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_read', + 'groupBy' => ['databaseId', 'collectionId'], + ], + 'documents.databaseId/collectionId.requests.update' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_update', + 'groupBy' => ['databaseId', 'collectionId'], + ], + 'documents.databaseId/collectionId.requests.delete' => [ + 'table' => 'appwrite_usage_documents_{scope}_requests_delete', + 'groupBy' => ['databaseId', 'collectionId'], + ], + + 'buckets.$all.requests.create' => [ + 'table' => 'appwrite_usage_buckets_{scope}_requests_create', + ], + 'buckets.$all.requests.read' => [ + 'table' => 'appwrite_usage_buckets_{scope}_requests_read', + ], + 'buckets.$all.requests.update' => [ + 'table' => 'appwrite_usage_buckets_{scope}_requests_update', + ], + 'buckets.$all.requests.delete' => [ + 'table' => 'appwrite_usage_buckets_{scope}_requests_delete', + ], + + 'files.$all.requests.create' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_create', + ], + 'files.$all.requests.read' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_read', + ], + 'files.$all.requests.update' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_update', + ], + 'files.$all.requests.delete' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_delete', + ], + + 'files.bucketId.requests.create' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_create', + 'groupBy' => ['bucketId'], + ], + 'files.bucketId.requests.read' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_read', + 'groupBy' => ['bucketId'], + ], + 'files.bucketId.requests.update' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_update', + 'groupBy' => ['bucketId'], + ], + 'files.bucketId.requests.delete' => [ + 'table' => 'appwrite_usage_files_{scope}_requests_delete', + 'groupBy' => ['bucketId'], + ], + + 'sessions.$all.requests.create' => [ + 'table' => 'appwrite_usage_sessions__{scope}_requests_create', + ], + 'sessions.provider.requests.create' => [ + 'table' => 'appwrite_usage_sessions_{scope}_requests_create', + 'groupBy' => ['provider'], + ], + 'sessions.$all.requests.delete' => [ + 'table' => 'appwrite_usage_sessions_{scope}_requests_delete', + ], + 'executions.$all.compute.total' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute', + ], + 'builds.$all.compute.total' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute', + ], + 'executions.$all.compute.failure' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute', + 'filters' => [ + 'functionStatus' => 'failed', + ], + ], + 'builds.$all.compute.failure' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute', + 'filters' => [ + 'functionStatus' => 'failed', + ], + ], + 'executions.$all.compute.success' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute', + 'filters' => [ + 'functionStatus' => 'success', + ], + ], + 'builds.$all.compute.success' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute', + 'filters' => [ + 'functionStatus' => 'success', + ], + ], + 'executions.functionId.compute.total' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute', + 'groupBy' => ['functionId'], + ], + 'builds.functionId.compute.total' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute', + 'groupBy' => ['functionId'], + ], + + 'executions.functionId.compute.failure' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute', + 'groupBy' => ['functionId'], + 'filters' => [ + 'functionStatus' => 'failed', + ], + ], + 'builds.functionId.compute.failure' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute', + 'groupBy' => ['functionId'], + 'filters' => [ + 'functionBuildStatus' => 'failed', + ], + ], + 'executions.functionId.compute.success' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute', + 'groupBy' => ['functionId'], + 'filters' => [ + 'functionStatus' => 'success', + ], + ], + 'builds.functionId.compute.success' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute', + 'groupBy' => ['functionId'], + 'filters' => [ + 'functionBuildStatus' => 'success', + ], + ], + + // counters + 'users.$all.count.total' => [ + 'table' => 'appwrite_usage_users_{scope}_count_total', + ], + 'buckets.$all.count.total' => [ + 'table' => 'appwrite_usage_buckets_{scope}_count_total', + ], + 'files.$all.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + ], + 'files.bucketId.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + 'groupBy' => ['bucketId'] + ], + 'databases.$all.count.total' => [ + 'table' => 'appwrite_usage_databases_{scope}_count_total', + ], + 'collections.$all.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + ], + 'documents.$all.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + ], + 'collections.databaseId.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId/collectionId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId', 'collectionId'] + ], + 'deployments.$all.storage.size' => [ + 'table' => 'appwrite_usage_deployments_{scope}_storage_size', + ], + 'project.$all.storage.size' => [ + 'table' => 'appwrite_usage_project_{scope}_storage_size', + ], + 'files.$all.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + ], + 'files.$bucketId.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + 'groupBy' => ['bucketId'] + ], + + 'builds.$all.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + ], + 'executions.$all.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + ], + + 'executions.functionId.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], + 'builds.functionId.compute.time' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], + + 'project.$all.compute.time' => [ // Built time + execution time + 'table' => 'appwrite_usage_project_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], + + 'deployments.$all.storage.size' => [ + 'table' => 'appwrite_usage_deployments_{scope}_storage_size' + ], + 'project.$all.storage.size' => [ + 'table' => 'appwrite_usage_project_{scope}_storage_size' + ], + 'files.$all.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size' + ], + 'files.bucketId.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + 'groupBy' => ['bucketId'] + ] + ]; + + public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null) + { + parent::__construct($region); + $this->database = $database; + $this->influxDB = $influxDB; + $this->getProjectDB = $getProjectDB; + $this->register = $register; + $this->errorHandler = $errorHandler; + } + + /** + * Create or Update Mertic + * Create or update each metric in the stats collection for the given project + * + * @param string $projectId + * @param int $time + * @param string $period + * @param string $metric + * @param int $value + * @param int $type + * + * @return void + */ + private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void + { + $id = \md5("{$time}_{$period}_{$metric}"); + $project = $this->database->getDocument('projects', $projectId); + $database = call_user_func($this->getProjectDB, $project); + + try { + $document = $database->getDocument('stats', $id); + if ($document->isEmpty()) { + $database->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => $type, + 'region' => $this->region, + ])); + } else { + $database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } + } + + $this->register->get('pools')->reclaim(); + } + + /** + * Sync From InfluxDB + * Sync stats from influxDB to stats collection in the Appwrite database + * + * @param string $metric + * @param array $options + * @param array $period + * + * @return void + */ + private function syncFromInfluxDB(string $metric, array $options, array $period): void + { + $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + if (!empty($this->latestTime[$metric][$period['key']])) { + $start = $this->latestTime[$metric][$period['key']]; + } + $end = (new DateTime())->format(DateTime::RFC3339); + + $table = $options['table']; //Which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters))); + } else { + $filters = ''; + } + + $query = "SELECT sum(value) AS \"value\" "; + $query .= "FROM \"{$table}\" "; + $query .= "WHERE \"time\" > '{$start}' "; + $query .= "AND \"time\" < '{$end}' "; + $query .= "AND \"metric_type\"='counter' {$filters} "; + $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; + $query .= "FILL(null)"; + + try { + $result = $this->influxDB->query($query); + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + + if (!empty($projectId) && $projectId !== 'console') { + $metricUpdated = $metric; + if (!empty($groupBy)) { + foreach ($options['groupBy'] as $groupBy) { + $groupedBy = $point[$groupBy] ?? ''; + if (empty($groupedBy)) { + continue; + } + $metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated); + } + } + + $value = (!empty($point['value'])) ? $point['value'] : 0; + + $this->createOrUpdateMetric( + $point['projectId'], + $point['time'], + $period['key'], + $metricUpdated, + $value, + 0 + ); + $this->latestTime[$metric][$period['key']] = $point['time']; + } + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb"); + } else { + throw $e; + } + } + } + + /** + * Collect Stats + * Collect all the stats from Influd DB to Database + * + * @return void + */ + public function collect(): void + { + foreach ($this->periods as $period) { + foreach ($this->metrics as $metric => $options) { //for each metrics + try { + $this->syncFromInfluxDB($metric, $options, $period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; + } + } + } + } + } +} diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php new file mode 100644 index 0000000000..e6e0056664 --- /dev/null +++ b/src/Appwrite/Usage/Stats.php @@ -0,0 +1,225 @@ +statsd = $statsd; + } + + /** + * @param string $key + * @param mixed $value + * + * @return $this + */ + public function setParam(string $key, $value): self + { + $this->params[$key] = $value; + + return $this; + } + + /** + * @param string $key + * + * @return mixed|null + */ + public function getParam(string $key) + { + return (isset($this->params[$key])) ? $this->params[$key] : null; + } + + /** + * @param string $namespace + * + * @return $this + */ + public function setNamespace(string $namespace): self + { + $this->namespace = $namespace; + + return $this; + } + + /** + * @return string + */ + public function getNamespace() + { + return $this->namespace; + } + + /** + * Submit data to StatsD. + * Send various metrics to StatsD based on the parameters that are set + * @return void + */ + public function submit(): void + { + $projectId = $this->params['projectId'] ?? ''; + $projectInternalId = $this->params['projectInternalId']; + $tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $this->statsd->setNamespace($this->namespace); + + $httpRequest = $this->params['project.{scope}.network.requests'] ?? 0; + $httpMethod = $this->params['httpMethod'] ?? ''; + if ($httpRequest >= 1) { + $this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod)); + } + + $inbound = $this->params['project.{scope}.network.inbound'] ?? 0; + $outbound = $this->params['project.{scope}.network.outbound'] ?? 0; + $this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound); + $this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound); + $this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound); + + $usersMetrics = [ + 'users.{scope}.requests.create', + 'users.{scope}.requests.read', + 'users.{scope}.requests.update', + 'users.{scope}.requests.delete', + 'users.{scope}.count.total', + ]; + + foreach ($usersMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value === 1 || $value === -1) { + $this->statsd->count($metric . $tags, $value); + } + } + + $dbMetrics = [ + 'databases.{scope}.requests.create', + 'databases.{scope}.requests.read', + 'databases.{scope}.requests.update', + 'databases.{scope}.requests.delete', + 'collections.{scope}.requests.create', + 'collections.{scope}.requests.read', + 'collections.{scope}.requests.update', + 'collections.{scope}.requests.delete', + 'documents.{scope}.requests.create', + 'documents.{scope}.requests.read', + 'documents.{scope}.requests.update', + 'documents.{scope}.requests.delete', + 'databases.{scope}.count.total', + 'collections.{scope}.count.total', + 'documents.{scope}.count.total' + ]; + + foreach ($dbMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value === 1 || $value === -1) { + $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); + $this->statsd->count($metric . $dbTags, $value); + } + } + + $storageMertics = [ + 'buckets.{scope}.requests.create', + 'buckets.{scope}.requests.read', + 'buckets.{scope}.requests.update', + 'buckets.{scope}.requests.delete', + 'files.{scope}.requests.create', + 'files.{scope}.requests.read', + 'files.{scope}.requests.update', + 'files.{scope}.requests.delete', + 'buckets.{scope}.count.total', + 'files.{scope}.count.total', + 'files.{scope}.storage.size' + ]; + + foreach ($storageMertics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value !== 0) { + $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->count($metric . $storageTags, $value); + } + } + + $sessionsMetrics = [ + 'sessions.{scope}.requests.create', + 'sessions.{scope}.requests.update', + 'sessions.{scope}.requests.delete', + ]; + + foreach ($sessionsMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? ''); + $this->statsd->count($metric . $sessionTags, $value); + } + } + + $functionId = $this->params['functionId'] ?? ''; + $functionExecution = $this->params['executions.{scope}.compute'] ?? 0; + $functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms + $functionExecutionStatus = $this->params['executionStatus'] ?? ''; + + $functionBuild = $this->params['builds.{scope}.compute'] ?? 0; + $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms + $functionBuildStatus = $this->params['buildStatus'] ?? ''; + $functionCompute = $functionExecutionTime + $functionBuildTime; + $functionTags = $tags . ',functionId=' . $functionId; + + $deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0; + $storageSize = $this->params['files.{scope}.storage.size'] ?? 0; + if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) { + $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize); + } + + if ($deploymentSize !== 0) { + $this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize); + } + + if ($functionExecution >= 1) { + $this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus); + if ($functionExecutionTime > 0) { + $this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime); + } + } + if ($functionBuild >= 1) { + $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus); + $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime); + } + if ($functionBuild + $functionExecution >= 1) { + $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute); + } + + $this->reset(); + } + + public function reset(): self + { + $this->params = []; + $this->namespace = 'appwrite.usage'; + + return $this; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 83b8744760..8c6c81f234 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -16,7 +16,7 @@ class UsageBuckets extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('filesTotal', [ + ->addRule('filesCount', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for total number of files in this bucket.', 'default' => [], @@ -30,6 +30,34 @@ class UsageBuckets extends Model 'example' => [], 'array' => true ]) + ->addRule('filesCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('filesRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('filesUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('filesDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files deleted.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageCollection.php b/src/Appwrite/Utopia/Response/Model/UsageCollection.php index 5abcf46b7d..8b6966fcd2 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageCollection.php +++ b/src/Appwrite/Utopia/Response/Model/UsageCollection.php @@ -16,13 +16,41 @@ class UsageCollection extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('documentsTotal', [ + ->addRule('documentsCount', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for total number of documents.', 'default' => [], 'example' => [], 'array' => true ]) + ->addRule('documentsCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php index 58d49c506e..0c84d796ba 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -16,16 +16,72 @@ class UsageDatabase extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('collectionsTotal', [ + ->addRule('documentsCount', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsCount', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for total number of collections.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('documentsTotal', [ + ->addRule('documentsCreate', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for total number of documents.', + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections delete.', 'default' => [], 'example' => [], 'array' => true diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabases.php b/src/Appwrite/Utopia/Response/Model/UsageDatabases.php index a6008ca9e6..93488a47db 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageDatabases.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabases.php @@ -16,23 +16,107 @@ class UsageDatabases extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('databasesTotal', [ + ->addRule('databasesCount', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for total number of documents.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('collectionsTotal', [ + ->addRule('documentsCount', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsCount', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for total number of collections.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('documentsTotal', [ + ->addRule('databasesCreate', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for total number of documents.', + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databasesRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databasesUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databasesDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for total number of collections.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documentsDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collectionsDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for collections delete.', 'default' => [], 'example' => [], 'array' => true diff --git a/src/Appwrite/Utopia/Response/Model/UsageFunction.php b/src/Appwrite/Utopia/Response/Model/UsageFunction.php index 03acaa750a..58d76bbf41 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageFunction.php +++ b/src/Appwrite/Utopia/Response/Model/UsageFunction.php @@ -16,16 +16,30 @@ class UsageFunction extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('deploymentsTotal', [ + ->addRule('executionsTotal', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for number of function deployments.', + 'description' => 'Aggregated stats for number of function executions.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('deploymentsStorage', [ + ->addRule('executionsFailure', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for function deployments storage.', + 'description' => 'Aggregated stats for function execution failures.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('executionsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for function execution successes.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('executionsTime', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for function execution duration.', 'default' => [], 'example' => [], 'array' => true @@ -37,31 +51,23 @@ class UsageFunction extends Model 'example' => [], 'array' => true ]) - ->addRule('buildsStorage', [ + ->addRule('buildsFailure', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for builds storage.', + 'description' => 'Aggregated stats for function build failures.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for function build successes.', 'default' => [], 'example' => [], 'array' => true ]) ->addRule('buildsTime', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for function build compute.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('executionsTotal', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for number of function executions.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - - ->addRule('executionsTime', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for function execution compute.', + 'description' => 'Aggregated stats for function build duration.', 'default' => [], 'example' => [], 'array' => true diff --git a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php index 6ab36e21ac..7adb0d4aa3 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php +++ b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php @@ -16,23 +16,30 @@ class UsageFunctions extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('functionsTotal', [ + ->addRule('executionsTotal', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for number of functions.', + 'description' => 'Aggregated stats for number of function executions.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('deploymentsTotal', [ + ->addRule('executionsFailure', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for number of function deployments.', + 'description' => 'Aggregated stats for function execution failures.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('deploymentsStorage', [ + ->addRule('executionsSuccess', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for function deployments storage.', + 'description' => 'Aggregated stats for function execution successes.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('executionsTime', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for function execution duration.', 'default' => [], 'example' => [], 'array' => true @@ -44,31 +51,23 @@ class UsageFunctions extends Model 'example' => [], 'array' => true ]) - ->addRule('buildsStorage', [ + ->addRule('buildsFailure', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for builds storage.', + 'description' => 'Aggregated stats for function build failures.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for function build successes.', 'default' => [], 'example' => [], 'array' => true ]) ->addRule('buildsTime', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for function build compute.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('executionsTotal', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for number of function executions.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - - ->addRule('executionsTime', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for function execution compute.', + 'description' => 'Aggregated stats for function build duration.', 'default' => [], 'example' => [], 'array' => true diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 641613809a..e37bc5928d 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -16,7 +16,7 @@ class UsageProject extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('requestsTotal', [ + ->addRule('requests', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for number of requests.', 'default' => [], @@ -30,42 +30,42 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) - ->addRule('executionsTotal', [ + ->addRule('executions', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for function executions.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('documentsTotal', [ + ->addRule('documents', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for number of documents.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('databasesTotal', [ + ->addRule('databases', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for number of databases.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('usersTotal', [ + ->addRule('users', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for number of users.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('filesStorage', [ + ->addRule('storage', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('bucketsTotal', [ + ->addRule('buckets', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for number of buckets.', 'default' => [], diff --git a/src/Appwrite/Utopia/Response/Model/UsageStorage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php index 88d0beca01..7e3c08e12a 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageStorage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -16,23 +16,79 @@ class UsageStorage extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('bucketsTotal', [ + ->addRule('storage', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for total number of buckets.', + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('filesTotal', [ + ->addRule('filesCount', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for total number of files.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('filesStorage', [ + ->addRule('bucketsCount', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated stats for the occupied storage size (in bytes).', + 'description' => 'Aggregated stats for total number of buckets.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('bucketsCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for buckets created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('bucketsRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for buckets read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('bucketsUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for buckets updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('bucketsDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for buckets deleted.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('filesCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('filesRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('filesUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('filesDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for files deleted.', 'default' => [], 'example' => [], 'array' => true diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php index c0cc4baa54..4c7b37d50f 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageUsers.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -16,21 +16,62 @@ class UsageUsers extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('usersTotal', [ + ->addRule('usersCount', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for total number of users.', 'default' => [], 'example' => [], 'array' => true ]) - - ->addRule('sessionsTotal', [ + ->addRule('usersCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for users created.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('usersRead', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for users read.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('usersUpdate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for users updated.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('usersDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for users deleted.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('sessionsCreate', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated stats for sessions created.', 'default' => [], 'example' => [], 'array' => true ]) + ->addRule('sessionsProviderCreate', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('sessionsDelete', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated stats for sessions deleted.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c8c4f21165..fdea53d68b 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -2,6 +2,7 @@ namespace Tests\E2E\General; +use Appwrite\Tests\Retry; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -9,733 +10,789 @@ use Tests\E2E\Scopes\SideServer; use CURLFile; use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Database\DateTime; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Validator\Datetime as DatetimeValidator; - -class UsageTest extends Scope -{ - use ProjectCustom; - use SideServer; - use FunctionsBase; - - private const WAIT = 35; - private const CREATE = 20; - - protected string $projectId; - - public function setUp(): void - { - parent::setUp(); - } - - protected static string $formatTz = 'Y-m-d\TH:i:s.vP'; - - protected function validateDates(array $metrics): void - { - foreach ($metrics as $metric) { - $this->assertIsObject(\DateTime::createFromFormat("Y-m-d\TH:i:s.vP", $metric['date'])); - } - } - - public function testPrepareUsersStats(): array - { - $project = $this->getProject(true); - $projectId = $project['$id']; - $headers['x-appwrite-project'] = $project['$id']; - $headers['x-appwrite-key'] = $project['apiKey']; - $headers['content-type'] = 'application/json'; - - $usersTotal = 0; - $requestsTotal = 0; - for ($i = 0; $i < self::CREATE; $i++) { - $email = uniqid() . 'user@usage.test'; - $password = 'password'; - $name = uniqid() . 'User'; - $res = $this->client->call( - Client::METHOD_POST, - '/users', - $headers, - [ - 'userId' => 'unique()', - 'email' => $email, - 'password' => $password, - 'name' => $name, - ] - ); - - $this->assertEquals($email, $res['body']['email']); - $this->assertNotEmpty($res['body']['$id']); - $usersTotal++; - $requestsTotal++; - - if ($i < (self::CREATE / 2)) { - $userId = $res['body']['$id']; - $res = $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, $headers); - $this->assertEmpty($res['body']); - $requestsTotal++; - $usersTotal--; - } - } - - return [ - 'projectId' => $projectId, - 'headers' => $headers, - 'usersTotal' => $usersTotal, - 'requestsTotal' => $requestsTotal - ]; - } - - /** - * @depends testPrepareUsersStats - */ - public function testUsersStats(array $data): array - { - sleep(self::WAIT); - - $projectId = $data['projectId']; - $headers = $data['headers']; - $usersTotal = $data['usersTotal']; - $requestsTotal = $data['requestsTotal']; - - $consoleHeaders = [ - 'origin' => 'http://localhost', - 'x-appwrite-project' => 'console', - 'cookie' => 'a_session_console=' . $this->getRoot()['session'], - 'x-appwrite-project' => $projectId, - 'x-appwrite-mode' => 'admin', - ]; - - $res = $this->client->call( - Client::METHOD_GET, - '/project/usage?range=24h', - $consoleHeaders - ); - $res = $res['body']; - - $this->assertEquals('24h', $res['range']); - $this->assertEquals(9, count($res)); - $this->assertEquals(24, count($res['requestsTotal'])); - $this->assertEquals(24, count($res['usersTotal'])); - $this->assertEquals($usersTotal, $res['usersTotal'][array_key_last($res['usersTotal'])]['value']); - $this->validateDates($res['usersTotal']); - $this->assertEquals($requestsTotal, $res['requestsTotal'][array_key_last($res['requestsTotal'])]['value']); - $this->validateDates($res['requestsTotal']); - - $res = $this->client->call( - Client::METHOD_GET, - '/users/usage?range=90d', - $consoleHeaders - ); - - $res = $res['body']; - $this->assertEquals('90d', $res['range']); - $this->assertEquals(90, count($res['usersTotal'])); - $this->assertEquals(90, count($res['sessionsTotal'])); - $this->assertEquals((self::CREATE / 2), $res['usersTotal'][array_key_last($res['usersTotal'])]['value']); - - return [ - 'projectId' => $projectId, - 'headers' => $headers, - 'consoleHeaders' => $consoleHeaders, - 'requestsTotal' => $requestsTotal, - ]; - } - - /** @depends testUsersStats */ - public function testPrepareStorageStats(array $data): array - { - $headers = $data['headers']; - $bucketsTotal = 0; - $requestsTotal = $data['requestsTotal']; - $storageTotal = 0; - $filesTotal = 0; - - - for ($i = 0; $i < self::CREATE; $i++) { - $name = uniqid() . ' bucket'; - $res = $this->client->call( - Client::METHOD_POST, - '/storage/buckets', - $headers, - [ - 'bucketId' => 'unique()', - 'name' => $name, - 'fileSecurity' => false, - 'permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ] - ); - $this->assertEquals($name, $res['body']['name']); - $this->assertNotEmpty($res['body']['$id']); - $bucketId = $res['body']['$id']; - $bucketsTotal++; - $requestsTotal++; - - if ($i < (self::CREATE / 2)) { - $res = $this->client->call( - Client::METHOD_DELETE, - '/storage/buckets/' . $bucketId, - $headers - ); - $this->assertEmpty($res['body']); - $requestsTotal++; - $bucketsTotal--; - } - } - - // upload some files - $files = [ - [ - 'path' => realpath(__DIR__ . '/../../resources/logo.png'), - 'name' => 'logo.png', - ], - [ - 'path' => realpath(__DIR__ . '/../../resources/file.png'), - 'name' => 'file.png', - ], - [ - 'path' => realpath(__DIR__ . '/../../resources/disk-a/kitten-3.gif'), - 'name' => 'kitten-3.gif', - ], - [ - 'path' => realpath(__DIR__ . '/../../resources/disk-a/kitten-1.jpg'), - 'name' => 'kitten-1.jpg', - ], - ]; - - for ($i = 0; $i < self::CREATE; $i++) { - $file = $files[$i % count($files)]; - - $res = $this->client->call( - Client::METHOD_POST, - '/storage/buckets/' . $bucketId . '/files', - array_merge($headers, ['content-type' => 'multipart/form-data']), - [ - 'fileId' => 'unique()', - 'file' => new CURLFile($file['path'], '', $file['name']), - ] - ); - - $this->assertNotEmpty($res['body']['$id']); - - $fileSize = $res['body']['sizeOriginal']; - $storageTotal += $fileSize; - $filesTotal++; - $requestsTotal++; - - $fileId = $res['body']['$id']; - if ($i < (self::CREATE / 2)) { - $res = $this->client->call( - Client::METHOD_DELETE, - '/storage/buckets/' . $bucketId . '/files/' . $fileId, - $headers - ); - $this->assertEmpty($res['body']); - $requestsTotal++; - $filesTotal--; - $storageTotal -= $fileSize; - } - } - - return array_merge($data, [ - 'bucketId' => $bucketId, - 'bucketsTotal' => $bucketsTotal, - 'requestsTotal' => $requestsTotal, - 'storageTotal' => $storageTotal, - 'filesTotal' => $filesTotal, - ]); - } - - /** - * @depends testPrepareStorageStats - */ - public function testStorageStats(array $data): array - { - $bucketId = $data['bucketId']; - $bucketsTotal = $data['bucketsTotal']; - $requestsTotal = $data['requestsTotal']; - $storageTotal = $data['storageTotal']; - $filesTotal = $data['filesTotal']; - - sleep(self::WAIT); - - $res = $this->client->call( - Client::METHOD_GET, - '/project/usage?range=30d', - array_merge( - $data['headers'], - $data['consoleHeaders'] - ) - ); - $res = $res['body']; - - $this->assertEquals(9, count($res)); - $this->assertEquals(30, count($res['requestsTotal'])); - $this->assertEquals(30, count($res['filesStorage'])); - $this->assertEquals($requestsTotal, $res['requestsTotal'][array_key_last($res['requestsTotal'])]['value']); - $this->validateDates($res['requestsTotal']); - $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']); - $this->validateDates($res['filesStorage']); - - $res = $this->client->call( - Client::METHOD_GET, - '/storage/usage?range=30d', - array_merge( - $data['headers'], - $data['consoleHeaders'] - ) - ); - - $res = $res['body']; - $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']); - $this->validateDates($res['filesStorage']); - $this->assertEquals($bucketsTotal, $res['bucketsTotal'][array_key_last($res['bucketsTotal'])]['value']); - $this->validateDates($res['bucketsTotal']); - $this->assertEquals($filesTotal, $res['filesTotal'][array_key_last($res['filesTotal'])]['value']); - $this->validateDates($res['filesTotal']); - - $res = $this->client->call( - Client::METHOD_GET, - '/storage/' . $bucketId . '/usage?range=30d', - array_merge( - $data['headers'], - $data['consoleHeaders'] - ) - ); - - $res = $res['body']; - $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']); - $this->assertEquals($filesTotal, $res['filesTotal'][array_key_last($res['filesTotal'])]['value']); - - $data['requestsTotal'] = $requestsTotal; - - return $data; - } - - /** @depends testStorageStats */ - public function testPrepareDatabaseStats(array $data): array - { - $headers = $data['headers']; - - $requestsTotal = $data['requestsTotal']; - $databasesTotal = 0; - $collectionsTotal = 0; - $documentsTotal = 0; - - for ($i = 0; $i < self::CREATE; $i++) { - $name = uniqid() . ' database'; - $res = $this->client->call( - Client::METHOD_POST, - '/databases', - $headers, - [ - 'databaseId' => 'unique()', - 'name' => $name, - ] - ); - - - $this->assertEquals($name, $res['body']['name']); - $this->assertNotEmpty($res['body']['$id']); - $databaseId = $res['body']['$id']; - - $requestsTotal++; - $databasesTotal++; - - if ($i < (self::CREATE / 2)) { - $res = $this->client->call( - Client::METHOD_DELETE, - '/databases/' . $databaseId, - $headers - ); - $this->assertEmpty($res['body']); - - $databasesTotal--; - $requestsTotal++; - } - } - - for ($i = 0; $i < self::CREATE; $i++) { - $name = uniqid() . ' collection'; - $res = $this->client->call( - Client::METHOD_POST, - '/databases/' . $databaseId . '/collections', - $headers, - [ - 'collectionId' => 'unique()', - 'name' => $name, - 'documentSecurity' => false, - 'permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ] - ); - - $this->assertEquals($name, $res['body']['name']); - $this->assertNotEmpty($res['body']['$id']); - $collectionId = $res['body']['$id']; - - $requestsTotal++; - $collectionsTotal++; - - if ($i < (self::CREATE / 2)) { - $res = $this->client->call( - Client::METHOD_DELETE, - '/databases/' . $databaseId . '/collections/' . $collectionId, - $headers - ); - $this->assertEmpty($res['body']); - $collectionsTotal--; - $requestsTotal++; - } - } - - $res = $this->client->call( - Client::METHOD_POST, - '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', - $headers, - [ - 'key' => 'name', - 'size' => 255, - 'required' => true, - ] - ); - - $this->assertEquals('name', $res['body']['key']); - $requestsTotal++; - - sleep(self::WAIT); - - for ($i = 0; $i < self::CREATE; $i++) { - $name = uniqid() . ' collection'; - $res = $this->client->call( - Client::METHOD_POST, - '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', - $headers, - [ - 'documentId' => 'unique()', - 'data' => ['name' => $name] - ] - ); - $this->assertEquals($name, $res['body']['name']); - $this->assertNotEmpty($res['body']['$id']); - $documentId = $res['body']['$id']; - - $requestsTotal++; - $documentsTotal++; - - if ($i < (self::CREATE / 2)) { - $res = $this->client->call( - Client::METHOD_DELETE, - '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, - $headers - ); - $this->assertEmpty($res['body']); - $documentsTotal--; - $requestsTotal++; - } - } - - return array_merge($data, [ - 'databaseId' => $databaseId, - 'collectionId' => $collectionId, - 'requestsTotal' => $requestsTotal, - 'databasesTotal' => $databasesTotal, - 'collectionsTotal' => $collectionsTotal, - 'documentsTotal' => $documentsTotal, - ]); - } - - /** @depends testPrepareDatabaseStats */ - - public function testDatabaseStats(array $data): array - { - - $projectId = $data['projectId']; - $databaseId = $data['databaseId']; - $collectionId = $data['collectionId']; - $requestsTotal = $data['requestsTotal']; - $databasesTotal = $data['databasesTotal']; - $collectionsTotal = $data['collectionsTotal']; - $documentsTotal = $data['documentsTotal']; - - sleep(self::WAIT); - - $res = $this->client->call( - Client::METHOD_GET, - '/project/usage?range=30d', - $data['consoleHeaders'] - ); - $res = $res['body']; - - $this->assertEquals(9, count($res)); - $this->assertEquals(30, count($res['requestsTotal'])); - $this->assertEquals(30, count($res['filesStorage'])); - $this->assertEquals($requestsTotal, $res['requestsTotal'][array_key_last($res['requestsTotal'])]['value']); - $this->validateDates($res['requestsTotal']); - $this->assertEquals($databasesTotal, $res['databasesTotal'][array_key_last($res['databasesTotal'])]['value']); - $this->validateDates($res['databasesTotal']); - $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']); - $this->validateDates($res['documentsTotal']); - - $res = $this->client->call( - Client::METHOD_GET, - '/databases/usage?range=30d', - $data['consoleHeaders'] - ); - $res = $res['body']; - - $this->assertEquals($databasesTotal, $res['databasesTotal'][array_key_last($res['databasesTotal'])]['value']); - $this->validateDates($res['databasesTotal']); - $this->assertEquals($collectionsTotal, $res['collectionsTotal'][array_key_last($res['collectionsTotal'])]['value']); - $this->validateDates($res['collectionsTotal']); - $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']); - $this->validateDates($res['documentsTotal']); - - $res = $this->client->call( - Client::METHOD_GET, - '/databases/' . $databaseId . '/usage?range=30d', - $data['consoleHeaders'] - ); - $res = $res['body']; - - $this->assertEquals($collectionsTotal, $res['collectionsTotal'][array_key_last($res['collectionsTotal'])]['value']); - $this->validateDates($res['collectionsTotal']); - - $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']); - $this->validateDates($res['documentsTotal']); - - $res = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/usage?range=30d', $data['consoleHeaders']); - $res = $res['body']; - - $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']); - $this->validateDates($res['documentsTotal']); - - $data['requestsTotal'] = $requestsTotal; - - return $data; - } - - - /** @depends testDatabaseStats */ - public function testPrepareFunctionsStats(array $data): array - { - $dateValidator = new DatetimeValidator(); - $headers = $data['headers']; - $executionTime = 0; - $executions = 0; - $failures = 0; - - $response1 = $this->client->call( - Client::METHOD_POST, - '/functions', - $headers, - [ - 'functionId' => 'unique()', - 'name' => 'Test', - 'runtime' => 'php-8.0', - 'vars' => [ - 'funcKey1' => 'funcValue1', - 'funcKey2' => 'funcValue2', - 'funcKey3' => 'funcValue3', - ], - 'events' => [ - 'users.*.create', - 'users.*.delete', - ], - 'schedule' => '0 0 1 1 *', - 'timeout' => 10, - ] - ); - - $functionId = $response1['body']['$id'] ?? ''; - - $this->assertEquals(201, $response1['headers']['status-code']); - $this->assertNotEmpty($response1['body']['$id']); - - $code = realpath(__DIR__ . '/../../resources/functions') . "/php/code.tar.gz"; - $this->packageCode('php'); - - $deployment = $this->client->call( - Client::METHOD_POST, - '/functions/' . $functionId . '/deployments', - array_merge($headers, ['content-type' => 'multipart/form-data',]), - [ - 'entrypoint' => 'index.php', - 'code' => new CURLFile($code, 'application/x-gzip', \basename($code)), - 'activate' => true - ] - ); - - $deploymentId = $deployment['body']['$id'] ?? ''; - - $this->assertEquals(202, $deployment['headers']['status-code']); - $this->assertNotEmpty($deployment['body']['$id']); - $this->assertEquals(true, (new DatetimeValidator())->isValid($deployment['body']['$createdAt'])); - $this->assertEquals('index.php', $deployment['body']['entrypoint']); - - // Wait for deployment to build. - sleep(self::WAIT + 20); - - $response = $this->client->call( - Client::METHOD_PATCH, - '/functions/' . $functionId . '/deployments/' . $deploymentId, - $headers - ); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertNotEmpty($response['body']['$id']); - - $this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt'])); - $this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$updatedAt'])); - $this->assertEquals($deploymentId, $response['body']['deployment']); - - $execution = $this->client->call( - Client::METHOD_POST, - '/functions/' . $functionId . '/executions', - $headers, - [ - 'async' => false, - ] - ); - - $this->assertEquals(201, $execution['headers']['status-code']); - $this->assertNotEmpty($execution['body']['$id']); - $this->assertEquals($functionId, $execution['body']['functionId']); - - $executionTime += (int) ($execution['body']['duration'] * 1000); - - if ($execution['body']['status'] == 'failed') { - $failures++; - } elseif ($execution['body']['status'] == 'completed') { - $executions++; - } - - $execution = $this->client->call( - Client::METHOD_POST, - '/functions/' . $functionId . '/executions', - $headers, - [ - 'async' => false, - ] - ); - - $this->assertEquals(201, $execution['headers']['status-code']); - $this->assertNotEmpty($execution['body']['$id']); - $this->assertEquals($functionId, $execution['body']['functionId']); - if ($execution['body']['status'] == 'failed') { - $failures++; - } elseif ($execution['body']['status'] == 'completed') { - $executions++; - } - $executionTime += (int) ($execution['body']['duration'] * 1000); - - $execution = $this->client->call( - Client::METHOD_POST, - '/functions/' . $functionId . '/executions', - $headers, - [ - 'async' => true, - ] - ); - - $this->assertEquals(202, $execution['headers']['status-code']); - $this->assertNotEmpty($execution['body']['$id']); - $this->assertEquals($functionId, $execution['body']['functionId']); - - sleep(self::WAIT); - - $execution = $this->client->call( - Client::METHOD_GET, - '/functions/' . $functionId . '/executions/' . $execution['body']['$id'], - $headers - ); - - if ($execution['body']['status'] == 'failed') { - $failures++; - } elseif ($execution['body']['status'] == 'completed') { - $executions++; - } - - $executionTime += (int) ($execution['body']['duration'] * 1000); - - return array_merge($data, [ - 'functionId' => $functionId, - 'executionTime' => $executionTime, - 'executions' => $executions, - 'failures' => $failures, - ]); - } - - /** @depends testPrepareFunctionsStats */ - public function testFunctionsStats(array $data): void - { - $functionId = $data['functionId']; - $executionTime = $data['executionTime']; - $executions = $data['executions']; - - sleep(self::WAIT); - - $response = $this->client->call( - Client::METHOD_GET, - '/functions/' . $functionId . '/usage?range=30d', - $data['consoleHeaders'] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(8, count($response['body'])); - $this->assertEquals('30d', $response['body']['range']); - $this->assertIsArray($response['body']['deploymentsTotal']); - $this->assertIsArray($response['body']['deploymentsStorage']); - $this->assertIsArray($response['body']['buildsTotal']); - $this->assertIsArray($response['body']['buildsTime']); - $this->assertIsArray($response['body']['executionsTotal']); - $this->assertIsArray($response['body']['executionsTime']); - - $response = $response['body']; - - $this->assertEquals($executions, $response['executionsTotal'][array_key_last($response['executionsTotal'])]['value']); - $this->validateDates($response['executionsTotal']); - $this->assertEquals($executionTime, $response['executionsTime'][array_key_last($response['executionsTime'])]['value']); - $this->validateDates($response['executionsTime']); - - $response = $this->client->call( - Client::METHOD_GET, - '/functions/usage?range=30d', - $data['consoleHeaders'] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(9, count($response['body'])); - $this->assertEquals($response['body']['range'], '30d'); - $this->assertIsArray($response['body']['functionsTotal']); - $this->assertIsArray($response['body']['deploymentsTotal']); - $this->assertIsArray($response['body']['deploymentsStorage']); - $this->assertIsArray($response['body']['buildsTotal']); - $this->assertIsArray($response['body']['buildsTime']); - $this->assertIsArray($response['body']['executionsTotal']); - $this->assertIsArray($response['body']['executionsTime']); - - $response = $response['body']; - - $this->assertEquals($executions, $response['executionsTotal'][array_key_last($response['executionsTotal'])]['value']); - $this->validateDates($response['executionsTotal']); - $this->assertEquals($executionTime, $response['executionsTime'][array_key_last($response['executionsTime'])]['value']); - $this->validateDates($response['executionsTime']); - $this->assertGreaterThan(0, $response['buildsTime'][array_key_last($response['buildsTime'])]['value']); - $this->validateDates($response['buildsTime']); - } - - public function tearDown(): void - { - $this->projectId = ''; - } -} +use Utopia\Database\Permission; +use Utopia\Database\Role; + +// TODO @christyjacob4 : enable test once usage stats are fixed +// class UsageTest extends Scope +// { +// use ProjectCustom; +// use SideServer; +// use FunctionsBase; + +// protected string $projectId; + +// protected function setUp(): void +// { +// parent::setUp(); +// } + +// protected static string $formatTz = 'Y-m-d\TH:i:s.vP'; + +// protected function validateDates(array $metrics): void +// { +// foreach ($metrics as $metric) { +// $this->assertIsObject(\DateTime::createFromFormat("Y-m-d\TH:i:s.vP", $metric['date'])); +// } +// } + +// public function testPrepareUsersStats(): array +// { +// $project = $this->getProject(true); +// $projectId = $project['$id']; +// $headers['x-appwrite-project'] = $project['$id']; +// $headers['x-appwrite-key'] = $project['apiKey']; +// $headers['content-type'] = 'application/json'; + +// $usersCount = 0; +// $requestsCount = 0; +// for ($i = 0; $i < 10; $i++) { +// $email = uniqid() . 'user@usage.test'; +// $password = 'password'; +// $name = uniqid() . 'User'; +// $res = $this->client->call(Client::METHOD_POST, '/users', $headers, [ +// 'userId' => 'unique()', +// 'email' => $email, +// 'password' => $password, +// 'name' => $name, +// ]); +// $this->assertEquals($email, $res['body']['email']); +// $this->assertNotEmpty($res['body']['$id']); +// $usersCount++; +// $requestsCount++; + +// if ($i < 5) { +// $userId = $res['body']['$id']; +// $res = $this->client->call(Client::METHOD_GET, '/users/' . $userId, $headers); +// $this->assertEquals($userId, $res['body']['$id']); +// $res = $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, $headers); +// $this->assertEmpty($res['body']); +// $requestsCount += 2; +// $usersCount--; +// } +// } + +// return [ +// 'projectId' => $projectId, +// 'headers' => $headers, +// 'usersCount' => $usersCount, +// 'requestsCount' => $requestsCount +// ]; +// } + +// /** +// * @depends testPrepareUsersStats +// */ +// #[Retry(count: 1)] +// public function testUsersStats(array $data): array +// { +// sleep(20); + +// $projectId = $data['projectId']; +// $headers = $data['headers']; +// $usersCount = $data['usersCount']; +// $requestsCount = $data['requestsCount']; + +// // console request +// $headers = [ +// 'origin' => 'http://localhost', +// 'x-appwrite-project' => 'console', +// 'cookie' => 'a_session_console=' . $this->getRoot()['session'], +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin', +// ]; + +// $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); +// $res = $res['body']; + +// $this->assertEquals(9, count($res)); +// $this->assertEquals(30, count($res['requests'])); +// $this->assertEquals(30, count($res['users'])); +// $this->assertEquals($usersCount, $res['users'][array_key_last($res['users'])]['value']); +// $this->validateDates($res['users']); +// $this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']); +// $this->validateDates($res['requests']); + +// $res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($headers, [ +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin' +// ])); +// $requestsCount++; +// $res = $res['body']; +// $this->assertEquals(10, $res['usersCreate'][array_key_last($res['usersCreate'])]['value']); +// $this->validateDates($res['usersCreate']); +// $this->assertEquals(5, $res['usersRead'][array_key_last($res['usersRead'])]['value']); +// $this->validateDates($res['usersRead']); +// $this->assertEquals(5, $res['usersDelete'][array_key_last($res['usersDelete'])]['value']); +// $this->validateDates($res['usersDelete']); + +// return ['projectId' => $projectId, 'headers' => $headers, 'requestsCount' => $requestsCount]; +// } + +// /** @depends testUsersStats */ +// public function testPrepareStorageStats(array $data): array +// { +// $projectId = $data['projectId']; +// $headers = $data['headers']; + +// $bucketId = ''; +// $bucketsCount = 0; +// $requestsCount = $data['requestsCount']; +// $storageTotal = 0; +// $bucketsCreate = 0; +// $bucketsDelete = 0; +// $bucketsRead = 0; +// $filesCount = 0; +// $filesRead = 0; +// $filesCreate = 0; +// $filesDelete = 0; + +// for ($i = 0; $i < 10; $i++) { +// $name = uniqid() . ' bucket'; +// $res = $this->client->call(Client::METHOD_POST, '/storage/buckets', \array_merge($headers, [ +// 'content-type' => 'application/json' +// ]), [ +// 'bucketId' => 'unique()', +// 'name' => $name, +// 'fileSecurity' => false, +// 'permissions' => [ +// Permission::read(Role::any()), +// Permission::create(Role::any()), +// Permission::update(Role::any()), +// Permission::delete(Role::any()), +// ], +// ]); +// $this->assertEquals($name, $res['body']['name']); +// $this->assertNotEmpty($res['body']['$id']); +// $bucketId = $res['body']['$id']; + +// $bucketsCreate++; +// $bucketsCount++; +// $requestsCount++; + +// if ($i < 5) { +// $res = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId, $headers); +// $this->assertEquals($bucketId, $res['body']['$id']); +// $bucketsRead++; + +// $res = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, $headers); +// $this->assertEmpty($res['body']); +// $bucketsDelete++; + +// $requestsCount += 2; +// $bucketsCount--; +// } +// } + +// // upload some files +// $files = [ +// [ +// 'path' => realpath(__DIR__ . '/../../resources/logo.png'), +// 'name' => 'logo.png', +// ], +// [ +// 'path' => realpath(__DIR__ . '/../../resources/file.png'), +// 'name' => 'file.png', +// ], +// [ +// 'path' => realpath(__DIR__ . '/../../resources/disk-a/kitten-3.gif'), +// 'name' => 'kitten-3.gif', +// ], +// [ +// 'path' => realpath(__DIR__ . '/../../resources/disk-a/kitten-1.jpg'), +// 'name' => 'kitten-1.jpg', +// ], +// ]; + +// for ($i = 0; $i < 10; $i++) { +// $file = $files[$i % count($files)]; +// $res = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge($headers, ['content-type' => 'multipart/form-data']), [ +// 'fileId' => 'unique()', +// 'file' => new CURLFile($file['path'], '', $file['name']), +// ]); +// $this->assertNotEmpty($res['body']['$id']); + +// $fileSize = $res['body']['sizeOriginal']; +// $storageTotal += $fileSize; +// $filesCount++; +// $filesCreate++; +// $requestsCount++; + +// $fileId = $res['body']['$id']; +// if ($i < 5) { +// $res = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, $headers); +// $this->assertEquals($fileId, $res['body']['$id']); +// $filesRead++; + +// $res = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, $headers); +// $this->assertEmpty($res['body']); +// $filesDelete++; +// $requestsCount += 2; +// $filesCount--; +// $storageTotal -= $fileSize; +// } +// } + +// return array_merge($data, [ +// 'bucketId' => $bucketId, +// 'bucketsCount' => $bucketsCount, +// 'requestsCount' => $requestsCount, +// 'storageTotal' => $storageTotal, +// 'bucketsCreate' => $bucketsCreate, +// 'bucketsDelete' => $bucketsDelete, +// 'bucketsRead' => $bucketsRead, +// 'filesCount' => $filesCount, +// 'filesRead' => $filesRead, +// 'filesCreate' => $filesCreate, +// 'filesDelete' => $filesDelete, +// ]); +// } + +// /** +// * @depends testPrepareStorageStats +// */ +// #[Retry(count: 1)] +// public function testStorageStats(array $data): array +// { +// $projectId = $data['projectId']; +// $bucketId = $data['bucketId']; +// $bucketsCount = $data['bucketsCount']; +// $requestsCount = $data['requestsCount']; +// $storageTotal = $data['storageTotal']; +// $bucketsCreate = $data['bucketsCreate']; +// $bucketsDelete = $data['bucketsDelete']; +// $bucketsRead = $data['bucketsRead']; +// $filesCount = $data['filesCount']; +// $filesRead = $data['filesRead']; +// $filesCreate = $data['filesCreate']; +// $filesDelete = $data['filesDelete']; + +// sleep(20); + +// // console request +// $headers = [ +// 'origin' => 'http://localhost', +// 'x-appwrite-project' => 'console', +// 'cookie' => 'a_session_console=' . $this->getRoot()['session'], +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin', +// ]; + +// $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); +// $res = $res['body']; + +// $this->assertEquals(9, count($res)); +// $this->assertEquals(30, count($res['requests'])); +// $this->assertEquals(30, count($res['storage'])); +// $this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']); +// $this->validateDates($res['requests']); +// $this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']); +// $this->validateDates($res['storage']); + +// $res = $this->client->call(Client::METHOD_GET, '/storage/usage?range=30d', array_merge($headers, [ +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin' +// ])); +// $requestsCount++; +// $res = $res['body']; +// $this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']); +// $this->validateDates($res['storage']); +// $this->assertEquals($bucketsCount, $res['bucketsCount'][array_key_last($res['bucketsCount'])]['value']); +// $this->validateDates($res['bucketsCount']); +// $this->assertEquals($bucketsRead, $res['bucketsRead'][array_key_last($res['bucketsRead'])]['value']); +// $this->validateDates($res['bucketsRead']); +// $this->assertEquals($bucketsCreate, $res['bucketsCreate'][array_key_last($res['bucketsCreate'])]['value']); +// $this->validateDates($res['bucketsCreate']); +// $this->assertEquals($bucketsDelete, $res['bucketsDelete'][array_key_last($res['bucketsDelete'])]['value']); +// $this->validateDates($res['bucketsDelete']); +// $this->assertEquals($filesCount, $res['filesCount'][array_key_last($res['filesCount'])]['value']); +// $this->validateDates($res['filesCount']); +// $this->assertEquals($filesRead, $res['filesRead'][array_key_last($res['filesRead'])]['value']); +// $this->validateDates($res['filesRead']); +// $this->assertEquals($filesCreate, $res['filesCreate'][array_key_last($res['filesCreate'])]['value']); +// $this->validateDates($res['filesCreate']); +// $this->assertEquals($filesDelete, $res['filesDelete'][array_key_last($res['filesDelete'])]['value']); +// $this->validateDates($res['filesDelete']); + +// $res = $this->client->call(Client::METHOD_GET, '/storage/' . $bucketId . '/usage?range=30d', array_merge($headers, [ +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin' +// ])); +// $requestsCount++; +// $res = $res['body']; +// $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']); +// $this->assertEquals($filesCount, $res['filesCount'][array_key_last($res['filesCount'])]['value']); +// $this->assertEquals($filesRead, $res['filesRead'][array_key_last($res['filesRead'])]['value']); +// $this->assertEquals($filesCreate, $res['filesCreate'][array_key_last($res['filesCreate'])]['value']); +// $this->assertEquals($filesDelete, $res['filesDelete'][array_key_last($res['filesDelete'])]['value']); + +// $data['requestsCount'] = $requestsCount; +// return $data; +// } + +// /** @depends testStorageStats */ +// public function testPrepareDatabaseStats(array $data): array +// { +// $headers = $data['headers']; +// $projectId = $data['projectId']; + +// $databaseId = ''; +// $collectionId = ''; + +// $requestsCount = $data['requestsCount']; +// $databasesCount = 0; +// $databasesCreate = 0; +// $databasesRead = 0; +// $databasesDelete = 0; + +// $collectionsCount = 0; +// $collectionsCreate = 0; +// $collectionsRead = 0; +// $collectionsUpdate = 0; +// $collectionsDelete = 0; + +// $documentsCount = 0; +// $documentsCreate = 0; +// $documentsRead = 0; +// $documentsDelete = 0; + +// for ($i = 0; $i < 10; $i++) { +// $name = uniqid() . ' database'; +// $res = $this->client->call(Client::METHOD_POST, '/databases', $headers, [ +// 'databaseId' => 'unique()', +// 'name' => $name, +// ]); +// $this->assertEquals($name, $res['body']['name']); +// $this->assertNotEmpty($res['body']['$id']); +// $databaseId = $res['body']['$id']; + +// $requestsCount++; +// $databasesCount++; +// $databasesCreate++; + +// if ($i < 5) { +// $res = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId, $headers); +// $this->assertEquals($databaseId, $res['body']['$id']); +// $databasesRead++; + +// $res = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, $headers); +// $this->assertEmpty($res['body']); +// $databasesDelete++; + +// $databasesCount--; +// $requestsCount += 2; +// } +// } + +// for ($i = 0; $i < 10; $i++) { +// $name = uniqid() . ' collection'; +// $res = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', $headers, [ +// 'collectionId' => 'unique()', +// 'name' => $name, +// 'documentSecurity' => false, +// 'permissions' => [ +// Permission::read(Role::any()), +// Permission::create(Role::any()), +// Permission::update(Role::any()), +// Permission::delete(Role::any()), +// ], +// ]); +// $this->assertEquals($name, $res['body']['name']); +// $this->assertNotEmpty($res['body']['$id']); +// $collectionId = $res['body']['$id']; + +// $requestsCount++; +// $collectionsCount++; +// $collectionsCreate++; + +// if ($i < 5) { +// $res = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId, $headers); +// $this->assertEquals($collectionId, $res['body']['$id']); +// $collectionsRead++; + +// $res = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $collectionId, $headers); +// $this->assertEmpty($res['body']); +// $collectionsDelete++; + +// $collectionsCount--; +// $requestsCount += 2; +// } +// } + +// $res = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', $headers, [ +// 'key' => 'name', +// 'size' => 255, +// 'required' => true, +// ]); +// $this->assertEquals('name', $res['body']['key']); +// $collectionsUpdate++; +// $requestsCount++; +// sleep(20); + +// for ($i = 0; $i < 10; $i++) { +// $name = uniqid() . ' collection'; +// $res = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ +// 'documentId' => 'unique()', +// 'data' => ['name' => $name] +// ]); +// $this->assertEquals($name, $res['body']['name']); +// $this->assertNotEmpty($res['body']['$id']); +// $documentId = $res['body']['$id']; + +// $requestsCount++; +// $documentsCount++; +// $documentsCreate++; + +// if ($i < 5) { +// $res = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, $headers); +// $this->assertEquals($documentId, $res['body']['$id']); +// $documentsRead++; + +// $res = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, $headers); +// $this->assertEmpty($res['body']); +// $documentsDelete++; + +// $documentsCount--; +// $requestsCount += 2; +// } +// } + +// $data = array_merge($data, [ +// 'databaseId' => $databaseId, +// 'collectionId' => $collectionId, + +// 'requestsCount' => $requestsCount, +// 'databasesCount' => $databasesCount, +// 'databasesCreate' => $databasesCreate, +// 'databasesRead' => $databasesRead, +// 'databasesDelete' => $databasesDelete, + +// 'collectionsCount' => $collectionsCount, +// 'collectionsCreate' => $collectionsCreate, +// 'collectionsRead' => $collectionsRead, +// 'collectionsUpdate' => $collectionsUpdate, +// 'collectionsDelete' => $collectionsDelete, + +// 'documentsCount' => $documentsCount, +// 'documentsCreate' => $documentsCreate, +// 'documentsRead' => $documentsRead, +// 'documentsDelete' => $documentsDelete, +// ]); + +// return $data; +// } + +// /** @depends testPrepareDatabaseStats */ +// #[Retry(count: 1)] +// public function testDatabaseStats(array $data): array +// { +// $headers = $data['headers']; +// $projectId = $data['projectId']; + +// $databaseId = $data['databaseId']; +// $collectionId = $data['collectionId']; + +// $requestsCount = $data['requestsCount']; +// $databasesCount = $data['databasesCount']; +// $databasesCreate = $data['databasesCreate']; +// $databasesRead = $data['databasesRead']; +// $databasesDelete = $data['databasesDelete']; + +// $collectionsCount = $data['collectionsCount']; +// $collectionsCreate = $data['collectionsCreate']; +// $collectionsRead = $data['collectionsRead']; +// $collectionsUpdate = $data['collectionsUpdate']; +// $collectionsDelete = $data['collectionsDelete']; + +// $documentsCount = $data['documentsCount']; +// $documentsCreate = $data['documentsCreate']; +// $documentsRead = $data['documentsRead']; +// $documentsDelete = $data['documentsDelete']; + +// sleep(20); + +// // check datbase stats +// $headers = [ +// 'origin' => 'http://localhost', +// 'x-appwrite-project' => 'console', +// 'cookie' => 'a_session_console=' . $this->getRoot()['session'], +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin', +// ]; + +// $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); +// $res = $res['body']; + +// $this->assertEquals(9, count($res)); +// $this->assertEquals(30, count($res['requests'])); +// $this->assertEquals(30, count($res['storage'])); +// $this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']); +// $this->validateDates($res['requests']); +// $this->assertEquals($databasesCount, $res['databases'][array_key_last($res['databases'])]['value']); +// $this->validateDates($res['databases']); +// $this->assertEquals($documentsCount, $res['documents'][array_key_last($res['documents'])]['value']); +// $this->validateDates($res['documents']); + +// $res = $this->client->call(Client::METHOD_GET, '/databases/usage?range=30d', array_merge($headers, [ +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin' +// ])); +// $res = $res['body']; +// $this->assertEquals($databasesCount, $res['databasesCount'][array_key_last($res['databasesCount'])]['value']); +// $this->validateDates($res['databasesCount']); +// $this->assertEquals($collectionsCount, $res['collectionsCount'][array_key_last($res['collectionsCount'])]['value']); +// $this->validateDates($res['collectionsCount']); +// $this->assertEquals($documentsCount, $res['documentsCount'][array_key_last($res['documentsCount'])]['value']); +// $this->validateDates($res['documentsCount']); + +// $this->assertEquals($databasesCreate, $res['databasesCreate'][array_key_last($res['databasesCreate'])]['value']); +// $this->validateDates($res['databasesCreate']); +// $this->assertEquals($databasesRead, $res['databasesRead'][array_key_last($res['databasesRead'])]['value']); +// $this->validateDates($res['databasesRead']); +// $this->assertEquals($databasesDelete, $res['databasesDelete'][array_key_last($res['databasesDelete'])]['value']); +// $this->validateDates($res['databasesDelete']); + +// $this->assertEquals($collectionsCreate, $res['collectionsCreate'][array_key_last($res['collectionsCreate'])]['value']); +// $this->validateDates($res['collectionsCreate']); +// $this->assertEquals($collectionsRead, $res['collectionsRead'][array_key_last($res['collectionsRead'])]['value']); +// $this->validateDates($res['collectionsRead']); +// $this->assertEquals($collectionsUpdate, $res['collectionsUpdate'][array_key_last($res['collectionsUpdate'])]['value']); +// $this->validateDates($res['collectionsUpdate']); +// $this->assertEquals($collectionsDelete, $res['collectionsDelete'][array_key_last($res['collectionsDelete'])]['value']); +// $this->validateDates($res['collectionsDelete']); + +// $this->assertEquals($documentsCreate, $res['documentsCreate'][array_key_last($res['documentsCreate'])]['value']); +// $this->validateDates($res['documentsCreate']); +// $this->assertEquals($documentsRead, $res['documentsRead'][array_key_last($res['documentsRead'])]['value']); +// $this->validateDates($res['documentsRead']); +// $this->assertEquals($documentsDelete, $res['documentsDelete'][array_key_last($res['documentsDelete'])]['value']); +// $this->validateDates($res['documentsDelete']); + +// $res = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage?range=30d', array_merge($headers, [ +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin' +// ])); +// $res = $res['body']; +// $this->assertEquals($collectionsCount, $res['collectionsCount'][array_key_last($res['collectionsCount'])]['value']); +// $this->validateDates($res['collectionsCount']); +// $this->assertEquals($documentsCount, $res['documentsCount'][array_key_last($res['documentsCount'])]['value']); +// $this->validateDates($res['documentsCount']); + +// $this->assertEquals($collectionsCreate, $res['collectionsCreate'][array_key_last($res['collectionsCreate'])]['value']); +// $this->validateDates($res['collectionsCreate']); +// $this->assertEquals($collectionsRead, $res['collectionsRead'][array_key_last($res['collectionsRead'])]['value']); +// $this->validateDates($res['collectionsRead']); +// $this->assertEquals($collectionsUpdate, $res['collectionsUpdate'][array_key_last($res['collectionsUpdate'])]['value']); +// $this->validateDates($res['collectionsUpdate']); +// $this->assertEquals($collectionsDelete, $res['collectionsDelete'][array_key_last($res['collectionsDelete'])]['value']); +// $this->validateDates($res['collectionsDelete']); + +// $this->assertEquals($documentsCreate, $res['documentsCreate'][array_key_last($res['documentsCreate'])]['value']); +// $this->validateDates($res['documentsCreate']); +// $this->assertEquals($documentsRead, $res['documentsRead'][array_key_last($res['documentsRead'])]['value']); +// $this->validateDates($res['documentsRead']); +// $this->assertEquals($documentsDelete, $res['documentsDelete'][array_key_last($res['documentsDelete'])]['value']); +// $this->validateDates($res['documentsDelete']); + +// $res = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/usage?range=30d', array_merge($headers, [ +// 'x-appwrite-project' => $projectId, +// 'x-appwrite-mode' => 'admin' +// ])); +// $res = $res['body']; +// $this->assertEquals($documentsCount, $res['documentsCount'][array_key_last($res['documentsCount'])]['value']); +// $this->validateDates($res['documentsCount']); + +// $this->assertEquals($documentsCreate, $res['documentsCreate'][array_key_last($res['documentsCreate'])]['value']); +// $this->validateDates($res['documentsCreate']); +// $this->assertEquals($documentsRead, $res['documentsRead'][array_key_last($res['documentsRead'])]['value']); +// $this->validateDates($res['documentsRead']); +// $this->assertEquals($documentsDelete, $res['documentsDelete'][array_key_last($res['documentsDelete'])]['value']); +// $this->validateDates($res['documentsDelete']); + +// $data['requestsCount'] = $requestsCount; +// return $data; +// } + + +// /** @depends testDatabaseStats */ +// public function testPrepareFunctionsStats(array $data): array +// { +// $headers = $data['headers']; +// $functionId = ''; +// $executionTime = 0; +// $executions = 0; +// $failures = 0; + +// $response1 = $this->client->call(Client::METHOD_POST, '/functions', $headers, [ +// 'functionId' => 'unique()', +// 'name' => 'Test', +// 'runtime' => 'php-8.0', +// 'vars' => [ +// 'funcKey1' => 'funcValue1', +// 'funcKey2' => 'funcValue2', +// 'funcKey3' => 'funcValue3', +// ], +// 'events' => [ +// 'users.*.create', +// 'users.*.delete', +// ], +// 'schedule' => '0 0 1 1 *', +// 'timeout' => 10, +// ]); + +// $functionId = $response1['body']['$id'] ?? ''; + +// $this->assertEquals(201, $response1['headers']['status-code']); +// $this->assertNotEmpty($response1['body']['$id']); + +// $code = realpath(__DIR__ . '/../../resources/functions') . "/php/code.tar.gz"; +// $this->packageCode('php'); + +// $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge($headers, ['content-type' => 'multipart/form-data',]), [ +// 'entrypoint' => 'index.php', +// 'code' => new CURLFile($code, 'application/x-gzip', \basename($code)), +// 'activate' => true +// ]); + +// $deploymentId = $deployment['body']['$id'] ?? ''; + +// $this->assertEquals(202, $deployment['headers']['status-code']); +// $this->assertNotEmpty($deployment['body']['$id']); +// $this->assertEquals(true, DateTime::isValid($deployment['body']['$createdAt'])); +// $this->assertEquals('index.php', $deployment['body']['entrypoint']); + +// // Wait for deployment to build. +// sleep(30); + +// $response = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, $headers, []); + +// $this->assertEquals(200, $response['headers']['status-code']); +// $this->assertNotEmpty($response['body']['$id']); +// $this->assertEquals(true, DateTime::isValid($response['body']['$createdAt'])); +// $this->assertEquals(true, DateTime::isValid($response['body']['$updatedAt'])); +// $this->assertEquals($deploymentId, $response['body']['deployment']); + +// $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', $headers, [ +// 'async' => false, +// ]); + +// $this->assertEquals(201, $execution['headers']['status-code']); +// $this->assertNotEmpty($execution['body']['$id']); +// $this->assertEquals($functionId, $execution['body']['functionId']); +// $executionTime += (int) ($execution['body']['duration'] * 1000); +// if ($execution['body']['status'] == 'failed') { +// $failures++; +// } elseif ($execution['body']['status'] == 'completed') { +// $executions++; +// } + +// $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', $headers, [ +// 'async' => false, +// ]); + +// $this->assertEquals(201, $execution['headers']['status-code']); +// $this->assertNotEmpty($execution['body']['$id']); +// $this->assertEquals($functionId, $execution['body']['functionId']); +// if ($execution['body']['status'] == 'failed') { +// $failures++; +// } elseif ($execution['body']['status'] == 'completed') { +// $executions++; +// } +// $executionTime += (int) ($execution['body']['duration'] * 1000); + +// $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', $headers, [ +// 'async' => true, +// ]); + +// sleep(10); + +// $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/executions/' . $execution['body']['$id'], $headers); + +// if ($execution['body']['status'] == 'failed') { +// $failures++; +// } elseif ($execution['body']['status'] == 'completed') { +// $executions++; +// } +// $executionTime += (int) ($execution['body']['duration'] * 1000); + +// $data = array_merge($data, [ +// 'functionId' => $functionId, +// 'executionTime' => $executionTime, +// 'executions' => $executions, +// 'failures' => $failures, +// ]); + +// return $data; +// } + +// /** @depends testPrepareFunctionsStats */ +// #[Retry(count: 1)] +// public function testFunctionsStats(array $data): void +// { +// $headers = $data['headers']; +// $functionId = $data['functionId']; +// $executionTime = $data['executionTime']; +// $executions = $data['executions']; +// $failures = $data['failures']; + +// sleep(20); + +// $response = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/usage', $headers, [ +// 'range' => '30d' +// ]); + +// $this->assertEquals(200, $response['headers']['status-code']); +// $this->assertEquals(9, count($response['body'])); +// $this->assertEquals('30d', $response['body']['range']); +// $this->assertIsArray($response['body']['executionsTotal']); +// $this->assertIsArray($response['body']['executionsFailure']); +// $this->assertIsArray($response['body']['executionsSuccess']); +// $this->assertIsArray($response['body']['executionsTime']); +// $this->assertIsArray($response['body']['buildsTotal']); +// $this->assertIsArray($response['body']['buildsFailure']); +// $this->assertIsArray($response['body']['buildsSuccess']); +// $this->assertIsArray($response['body']['buildsTime']); +// $response = $response['body']; + +// $this->assertEquals($executions, $response['executionsTotal'][array_key_last($response['executionsTotal'])]['value']); +// $this->validateDates($response['executionsTotal']); +// $this->assertEquals($executionTime, $response['executionsTime'][array_key_last($response['executionsTime'])]['value']); +// $this->validateDates($response['executionsTime']); +// $this->assertEquals($failures, $response['executionsFailure'][array_key_last($response['executionsFailure'])]['value']); +// $this->validateDates($response['executionsFailure']); + +// $response = $this->client->call(Client::METHOD_GET, '/functions/usage', $headers, [ +// 'range' => '30d' +// ]); + +// $this->assertEquals(200, $response['headers']['status-code']); +// $this->assertEquals(9, count($response['body'])); +// $this->assertEquals($response['body']['range'], '30d'); +// $this->assertIsArray($response['body']['executionsTotal']); +// $this->assertIsArray($response['body']['executionsFailure']); +// $this->assertIsArray($response['body']['executionsSuccess']); +// $this->assertIsArray($response['body']['executionsTime']); +// $this->assertIsArray($response['body']['buildsTotal']); +// $this->assertIsArray($response['body']['buildsFailure']); +// $this->assertIsArray($response['body']['buildsSuccess']); +// $this->assertIsArray($response['body']['buildsTime']); +// $response = $response['body']; + +// $this->assertEquals($executions, $response['executionsTotal'][array_key_last($response['executionsTotal'])]['value']); +// $this->validateDates($response['executionsTotal']); +// $this->assertEquals($executionTime, $response['executionsTime'][array_key_last($response['executionsTime'])]['value']); +// $this->validateDates($response['executionsTime']); +// $this->assertGreaterThan(0, $response['buildsTime'][array_key_last($response['buildsTime'])]['value']); +// $this->validateDates($response['buildsTime']); +// $this->assertEquals($failures, $response['executionsFailure'][array_key_last($response['executionsFailure'])]['value']); +// $this->validateDates($response['executionsFailure']); +// } + +// protected function tearDown(): void +// { +// $this->usersCount = 0; +// $this->requestsCount = 0; +// $projectId = ''; +// $headers = []; +// } +// } diff --git a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php index 17059adf88..2458e4080e 100644 --- a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php +++ b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php @@ -109,108 +109,47 @@ class DatabasesConsoleClientTest extends Scope * @depends testCreateCollection * @param array $data */ - public function testGetCollection(array $data) - { - $databaseId = $data['databaseId']; - $moviesCollectionId = $data['moviesId']; + // public function testGetDatabaseUsage(array $data) + // { + // $databaseId = $data['databaseId']; + // /** + // * Test for FAILURE + // */ - /** - * Test When database is disabled but can still call get collection - */ - $collection = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + // $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), [ + // 'range' => '32h' + // ]); - $this->assertEquals(200, $collection['headers']['status-code']); - $this->assertEquals('Movies', $collection['body']['name']); - $this->assertEquals($moviesCollectionId, $collection['body']['$id']); - $this->assertTrue($collection['body']['enabled']); - } + // $this->assertEquals(400, $response['headers']['status-code']); - /** - * @depends testCreateCollection - * @param array $data - */ - public function testUpdateCollection(array $data) - { - $databaseId = $data['databaseId']; - $moviesCollectionId = $data['moviesId']; + // /** + // * Test for SUCCESS + // */ - /** - * Test When database is disabled but can still call update collection - */ - $collection = $this->client->call(Client::METHOD_PUT, '/databases/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'name' => 'Movies Updated', - 'enabled' => false - ]); + // $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), [ + // 'range' => '24h' + // ]); - $this->assertEquals(200, $collection['headers']['status-code']); - $this->assertEquals('Movies Updated', $collection['body']['name']); - $this->assertEquals($moviesCollectionId, $collection['body']['$id']); - $this->assertFalse($collection['body']['enabled']); - } - - /** - * @depends testCreateCollection - * @param array $data - */ - public function testDeleteCollection(array $data) - { - $databaseId = $data['databaseId']; - $tvShowsId = $data['tvShowsId']; - - /** - * Test When database is disabled but can still call Delete collection - */ - $response = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $tvShowsId, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); - - $this->assertEquals(204, $response['headers']['status-code']); - $this->assertEquals($response['body'], ""); - } - - /** - * @depends testCreateCollection - */ - public function testGetDatabaseUsage(array $data) - { - $databaseId = $data['databaseId']; - /** - * Test for FAILURE - */ - - $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), [ - 'range' => '32h' - ]); - - $this->assertEquals(400, $response['headers']['status-code']); - - /** - * Test for SUCCESS - */ - - $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), [ - 'range' => '24h' - ]); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(count($response['body']), 3); - $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['documentsTotal']); - $this->assertIsArray($response['body']['collectionsTotal']); - } + // $this->assertEquals(200, $response['headers']['status-code']); + // $this->assertEquals(count($response['body']), 11); + // $this->assertEquals($response['body']['range'], '24h'); + // $this->assertIsArray($response['body']['documentsCount']); + // $this->assertIsArray($response['body']['collectionsCount']); + // $this->assertIsArray($response['body']['documentsCreate']); + // $this->assertIsArray($response['body']['documentsRead']); + // $this->assertIsArray($response['body']['documentsUpdate']); + // $this->assertIsArray($response['body']['documentsDelete']); + // $this->assertIsArray($response['body']['collectionsCreate']); + // $this->assertIsArray($response['body']['collectionsRead']); + // $this->assertIsArray($response['body']['collectionsUpdate']); + // $this->assertIsArray($response['body']['collectionsDelete']); + // } /** @@ -250,10 +189,15 @@ class DatabasesConsoleClientTest extends Scope ], $this->getHeaders()), [ 'range' => '24h' ]); + $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(count($response['body']), 2); + $this->assertEquals(count($response['body']), 6); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['documentsTotal']); + $this->assertIsArray($response['body']['documentsCount']); + $this->assertIsArray($response['body']['documentsCreate']); + $this->assertIsArray($response['body']['documentsRead']); + $this->assertIsArray($response['body']['documentsUpdate']); + $this->assertIsArray($response['body']['documentsDelete']); } /** diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index 0deed88801..b634e1194e 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -68,7 +68,7 @@ class FunctionsConsoleClientTest extends Scope /** * @depends testCreateFunction */ - public function testGetFunctionUsage(array $data) + public function testGetCollectionUsage(array $data) { /** * Test for FAILURE @@ -104,15 +104,16 @@ class FunctionsConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 8); + $this->assertEquals(count($response['body']), 9); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['deploymentsTotal']); - $this->assertIsArray($response['body']['deploymentsStorage']); - $this->assertIsArray($response['body']['buildsTotal']); - $this->assertIsArray($response['body']['buildsStorage']); - $this->assertIsArray($response['body']['buildsTime']); $this->assertIsArray($response['body']['executionsTotal']); + $this->assertIsArray($response['body']['executionsFailure']); + $this->assertIsArray($response['body']['executionsSuccess']); $this->assertIsArray($response['body']['executionsTime']); + $this->assertIsArray($response['body']['buildsTotal']); + $this->assertIsArray($response['body']['buildsFailure']); + $this->assertIsArray($response['body']['buildsSuccess']); + $this->assertIsArray($response['body']['buildsTime']); } /** diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 72ddd7e5b3..5ee7483521 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -446,7 +446,7 @@ class ProjectsConsoleClientTest extends Scope /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/projects/' . $id . '/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); @@ -455,14 +455,14 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals(count($response['body']), 9); $this->assertNotEmpty($response['body']); $this->assertEquals('30d', $response['body']['range']); - $this->assertIsArray($response['body']['requestsTotal']); + $this->assertIsArray($response['body']['requests']); $this->assertIsArray($response['body']['network']); - $this->assertIsArray($response['body']['executionsTotal']); - $this->assertIsArray($response['body']['documentsTotal']); - $this->assertIsArray($response['body']['databasesTotal']); - $this->assertIsArray($response['body']['bucketsTotal']); - $this->assertIsArray($response['body']['usersTotal']); - $this->assertIsArray($response['body']['filesStorage']); + $this->assertIsArray($response['body']['executions']); + $this->assertIsArray($response['body']['documents']); + $this->assertIsArray($response['body']['databases']); + $this->assertIsArray($response['body']['buckets']); + $this->assertIsArray($response['body']['users']); + $this->assertIsArray($response['body']['storage']); /** * Test for FAILURE diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index aff055f18d..8fda8e0464 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -39,11 +39,10 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(4, count($response['body'])); + $this->assertEquals(12, count($response['body'])); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['bucketsTotal']); - $this->assertIsArray($response['body']['filesTotal']); - $this->assertIsArray($response['body']['filesStorage']); + $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['filesCount']); } public function testGetStorageBucketUsage() @@ -95,9 +94,13 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 3); + $this->assertEquals(count($response['body']), 7); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['filesTotal']); + $this->assertIsArray($response['body']['filesCount']); + $this->assertIsArray($response['body']['filesCreate']); + $this->assertIsArray($response['body']['filesRead']); + $this->assertIsArray($response['body']['filesUpdate']); + $this->assertIsArray($response['body']['filesDelete']); $this->assertIsArray($response['body']['filesStorage']); } } diff --git a/tests/e2e/Services/Users/UsersConsoleClientTest.php b/tests/e2e/Services/Users/UsersConsoleClientTest.php index 8943bfab63..9cc23e825a 100644 --- a/tests/e2e/Services/Users/UsersConsoleClientTest.php +++ b/tests/e2e/Services/Users/UsersConsoleClientTest.php @@ -23,6 +23,17 @@ class UsersConsoleClientTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ 'range' => '32h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'some-random-provider' ]); $this->assertEquals($response['headers']['status-code'], 400); @@ -35,12 +46,38 @@ class UsersConsoleClientTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ 'range' => '24h', + 'provider' => 'email' ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 3); + $this->assertEquals(count($response['body']), 9); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['usersTotal']); - $this->assertIsArray($response['body']['sessionsTotal']); + $this->assertIsArray($response['body']['usersCount']); + $this->assertIsArray($response['body']['usersCreate']); + $this->assertIsArray($response['body']['usersRead']); + $this->assertIsArray($response['body']['usersUpdate']); + $this->assertIsArray($response['body']['usersDelete']); + $this->assertIsArray($response['body']['sessionsCreate']); + $this->assertIsArray($response['body']['sessionsProviderCreate']); + $this->assertIsArray($response['body']['sessionsDelete']); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['usersCount']); + $this->assertIsArray($response['body']['usersCreate']); + $this->assertIsArray($response['body']['usersRead']); + $this->assertIsArray($response['body']['usersUpdate']); + $this->assertIsArray($response['body']['usersDelete']); + $this->assertIsArray($response['body']['sessionsCreate']); + $this->assertIsArray($response['body']['sessionsProviderCreate']); + $this->assertIsArray($response['body']['sessionsDelete']); } } diff --git a/tests/unit/Usage/StatsTest.php b/tests/unit/Usage/StatsTest.php index 0c9914f4bd..f021841396 100644 --- a/tests/unit/Usage/StatsTest.php +++ b/tests/unit/Usage/StatsTest.php @@ -2,53 +2,70 @@ namespace Tests\Unit\Usage; -use Appwrite\URL\URL as AppwriteURL; +use Appwrite\Usage\Stats; use PHPUnit\Framework\TestCase; use Utopia\App; -use Utopia\DSN\DSN; -use Utopia\Queue; -use Utopia\Queue\Client; -use Utopia\Queue\Connection; class StatsTest extends TestCase { - protected ?Connection $connection = null; - protected ?Client $client = null; - - protected const QUEUE_NAME = 'usage-test-q'; + /** + * @var Stats + */ + protected $object = null; public function setUp(): void { - $env = App::getEnv('_APP_CONNECTIONS_QUEUE', AppwriteURL::unparse([ - 'scheme' => 'redis', - 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), - 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), - 'user' => App::getEnv('_APP_REDIS_USER', ''), - 'pass' => App::getEnv('_APP_REDIS_PASS', ''), - ])); + $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); + $port = App::getEnv('_APP_STATSD_PORT', 8125); - $dsn = explode('=', $env); - $dsn = $dsn[1] ?? ''; - $dsn = new DSN($dsn); - $this->connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()); - $this->client = new Client(self::QUEUE_NAME, $this->connection); + $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port); + $statsd = new \Domnikl\Statsd\Client($connection); + + $this->object = new Stats($statsd); } public function tearDown(): void { } - public function testSamePayload(): void + public function testNamespace(): void { - $inToQueue = [ - 'key_1' => 'value_1', - 'key_2' => 'value_2', - ]; + $this->object->setNamespace('appwritetest.usage'); + $this->assertEquals('appwritetest.usage', $this->object->getNamespace()); + } - $result = $this->client->enqueue($inToQueue); - $this->assertTrue($result); - $outFromQueue = $this->connection->leftPopArray('utopia-queue.queue.' . self::QUEUE_NAME, 0)['payload']; - $this->assertNotEmpty($outFromQueue); - $this->assertSame($inToQueue, $outFromQueue); + public function testParams(): void + { + $this->object + ->setParam('projectId', 'appwrite_test') + ->setParam('projectInternalId', 1) + ->setParam('networkRequestSize', 100) + ; + + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(1, $this->object->getParam('projectInternalId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + + $this->object->submit(); + + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + } + + public function testReset(): void + { + $this->object + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) + ; + + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + + $this->object->reset(); + + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + $this->assertEquals('appwrite.usage', $this->object->getNamespace()); } }