diff --git a/.env b/.env index 2373b618c2..b915f91516 100644 --- a/.env +++ b/.env @@ -78,7 +78,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=30 +_APP_USAGE_AGGREGATION_INTERVAL=20 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/app/config/collections.php b/app/config/collections.php index 3bc58eff56..e2f3c11be7 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -34,28 +34,6 @@ $commonCollections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => 'resourceType', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 255, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('mimeType'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2 - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ '$id' => 'accessedAt', 'type' => Database::VAR_DATETIME, @@ -1349,10 +1327,10 @@ $commonCollections = [ ] ], - 'stats' => [ + 'stats_v2' => [ '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('stats'), - 'name' => 'stats', + '$id' => ID::custom('stats_v2'), + 'name' => 'stats_v2', 'attributes' => [ [ '$id' => ID::custom('metric'), diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 17b2984471..68d2a544d8 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1791,7 +1791,7 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), + 'total' => $audit->countLogsByUser($user->getId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index 5f2ea4c2da..e0d967eb00 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -76,7 +76,7 @@ $getUserGitHub = function (string $userId, Document $project, Database $dbForPro } if (empty($gitHubSession)) { - throw new Exception(Exception::USER_SESSION_NOT_FOUND, 'GitHub session not found.'); + throw new Exception(Exception::GENERAL_UNKNOWN, 'GitHub session not found.'); } $provider = $gitHubSession->getAttribute('provider', ''); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 5fdb2edb2f..8684a7fae8 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3563,7 +3563,7 @@ App::get('/v1/databases/usage') Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -3571,7 +3571,7 @@ App::get('/v1/databases/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), @@ -3647,7 +3647,7 @@ App::get('/v1/databases/:databaseId/usage') Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -3655,7 +3655,7 @@ App::get('/v1/databases/:databaseId/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), @@ -3733,7 +3733,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -3741,7 +3741,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 45df54b709..d3913d180d 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -484,7 +484,7 @@ App::get('/v1/functions/:functionId/usage') Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -492,7 +492,7 @@ App::get('/v1/functions/:functionId/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), @@ -576,7 +576,7 @@ App::get('/v1/functions/usage') Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -584,7 +584,7 @@ App::get('/v1/functions/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index f7fea04fee..c007d629b6 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -659,60 +659,6 @@ App::get('/v1/health/queue/functions') $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); -App::get('/v1/health/queue/usage') - ->desc('Get usage queue') - ->groups(['api', 'health']) - ->label('scope', 'health.read') - ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) - ->label('sdk.namespace', 'health') - ->label('sdk.method', 'getQueueUsage') - ->label('sdk.description', '/docs/references/health/get-queue-usage.md') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('queue') - ->inject('response') - ->action(function (int|string $threshold, Connection $queue, Response $response) { - $threshold = \intval($threshold); - - $client = new Client(Event::USAGE_QUEUE_NAME, $queue); - $size = $client->getQueueSize(); - - if ($size >= $threshold) { - throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); - } - - $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); - }); - -App::get('/v1/health/queue/usage-dump') - ->desc('Get usage dump queue') - ->groups(['api', 'health']) - ->label('scope', 'health.read') - ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) - ->label('sdk.namespace', 'health') - ->label('sdk.method', 'getQueueUsage') - ->label('sdk.description', '/docs/references/health/get-queue-usage-dump.md') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('queue') - ->inject('response') - ->action(function (int|string $threshold, Connection $queue, Response $response) { - $threshold = \intval($threshold); - - $client = new Client(Event::USAGE_DUMP_QUEUE_NAME, $queue); - $size = $client->getQueueSize(); - - if ($size >= $threshold) { - throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); - } - - $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); - }); - App::get('/v1/health/storage/local') ->desc('Get local storage') ->groups(['api', 'health']) @@ -850,7 +796,6 @@ App::get('/v1/health/queue/failed/:name') Event::MAILS_QUEUE_NAME, Event::FUNCTIONS_QUEUE_NAME, Event::USAGE_QUEUE_NAME, - Event::USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME, Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index a067c45588..47b94d3093 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -73,7 +73,7 @@ App::get('/v1/project/usage') Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) { foreach ($metrics['total'] as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -81,7 +81,7 @@ App::get('/v1/project/usage') } foreach ($metrics['period'] as $metric) { - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::greaterThanEqual('time', $firstDay), @@ -116,7 +116,7 @@ App::get('/v1/project/usage') $id = $function->getId(); $name = $function->getAttribute('name'); $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS); - $value = $dbForProject->findOne('stats', [ + $value = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -132,7 +132,7 @@ App::get('/v1/project/usage') $id = $bucket->getId(); $name = $bucket->getAttribute('name'); $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); - $value = $dbForProject->findOne('stats', [ + $value = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index e4798fb51e..efe2515468 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1467,7 +1467,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') if ($deviceDeleted) { $queueForDeletes ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) - ->setResourceType('bucket/' . $bucket->getId()) ->setResource('file/' . $fileId) ; @@ -1525,7 +1524,7 @@ App::get('/v1/storage/usage') $total = []; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -1533,7 +1532,7 @@ App::get('/v1/storage/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), @@ -1610,7 +1609,7 @@ App::get('/v1/storage/:bucketId/usage') Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -1618,7 +1617,7 @@ App::get('/v1/storage/:bucketId/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index bbeef96f14..5ce2263f47 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -615,9 +615,6 @@ App::get('/v1/users/:userId/logs') $output[$i] = new Document([ 'event' => $log['event'], - 'userId' => ID::custom($log['data']['userId']), - 'userEmail' => $log['data']['userEmail'] ?? null, - 'userName' => $log['data']['userName'] ?? null, 'ip' => $log['ip'], 'time' => $log['time'], 'osCode' => $os['osCode'], @@ -646,7 +643,7 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), + 'total' => $audit->countLogsByUser($user->getId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -1266,7 +1263,7 @@ App::get('/v1/users/usage') Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { foreach ($metrics as $count => $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -1274,7 +1271,7 @@ App::get('/v1/users/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 0423741e6d..df6ec002cb 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -252,22 +252,24 @@ App::init() ; $useCache = $route->getLabel('cache', false); + if ($useCache) { $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); - $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); $timestamp = 60 * 60 * 24 * 30; $data = $cache->load($key, $timestamp); - if (!empty($data) && !$cacheLog->isEmpty()) { - $parts = explode('/', $cacheLog->getAttribute('resourceType')); + if (!empty($data)) { + $data = json_decode($data, true); + $parts = explode('/', $data['resourceType']); $type = $parts[0] ?? null; if ($type === 'bucket') { $bucketId = $parts[1] ?? null; - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -279,12 +281,11 @@ App::init() $fileSecurity = $bucket->getAttribute('fileSecurity', false); $validator = new Authorization(Database::PERMISSION_READ); $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid) { throw new Exception(Exception::USER_UNAUTHORIZED); } - $parts = explode('/', $cacheLog->getAttribute('resource')); + $parts = explode('/', $data['resource']); $fileId = $parts[1] ?? null; if ($fileSecurity && !$valid) { @@ -301,8 +302,8 @@ App::init() $response ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT') ->addHeader('X-Appwrite-Cache', 'hit') - ->setContentType($cacheLog->getAttribute('mimeType')) - ->send($data) + ->setContentType($data['contentType']) + ->send(base64_decode($data['payload'])) ; } else { $response->addHeader('X-Appwrite-Cache', 'miss'); @@ -533,6 +534,7 @@ App::shutdown() if ($useCache) { $resource = $resourceType = null; $data = $response->getPayload(); + if (!empty($data['payload'])) { $pattern = $route->getLabel('cache.resource', null); if (!empty($pattern)) { @@ -544,17 +546,22 @@ App::shutdown() $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user); } - $key = md5($request->getURI() . '*' . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER; - $signature = md5($data['payload']); - $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); + $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); + $data = json_encode([ + 'resourceType' => $resourceType, + 'resource' => $resource, + 'contentType' => $response->getContentType(), + 'payload' => base64_encode($data['payload']), + ]) ; + + $signature = md5($data); + $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $accessedAt = $cacheLog->getAttribute('accessedAt', ''); $now = DateTime::now(); if ($cacheLog->isEmpty()) { Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([ '$id' => $key, 'resource' => $resource, - 'resourceType' => $resourceType, - 'mimeType' => $response->getContentType(), 'accessedAt' => $now, 'signature' => $signature, ]))); @@ -567,7 +574,7 @@ App::shutdown() $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); - $cache->save($key, $data['payload']); + $cache->save($key, $data); } } } diff --git a/app/init.php b/app/init.php index f27da88d24..80c3670eec 100644 --- a/app/init.php +++ b/app/init.php @@ -112,7 +112,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 330; +const APP_CACHE_BUSTER = 329; const APP_VERSION_STABLE = '1.4.13'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; diff --git a/docker-compose.yml b/docker-compose.yml index ed912dc8e2..de71e3937a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -928,7 +928,6 @@ services: # - appwrite # volumes: # - appwrite-uploads:/storage/uploads - # Dev Tools Start ------------------------------------------------------------------------------------------ # # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack @@ -937,9 +936,8 @@ services: # # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI. # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks - # Redis Insight - A nice UI for exploring Redis data - # Adminer - A nice UI for exploring MariaDB data - # GraphQl Explorer - A nice UI for exploring GraphQL API + # RedisCommander - A nice UI for exploring Redis data + # Webgrind - A nice UI for exploring and debugging code-level stuff maildev: # used mainly for dev tests image: appwrite/mailcatcher:1.0.0 @@ -969,15 +967,21 @@ services: networks: - appwrite - redis-insight: - image: redis/redisinsight:latest - restart: unless-stopped - networks: - - appwrite - environment: - - REDIS_HOSTS=redis - ports: - - "8081:5540" + # redis-commander: + # image: rediscommander/redis-commander:latest + # restart: unless-stopped + # networks: + # - appwrite + # environment: + # - REDIS_HOSTS=redis + # ports: + # - "8081:8081" + # webgrind: + # image: 'jokkedk/webgrind:latest' + # volumes: + # - './debug:/tmp' + # ports: + # - '3001:80' graphql-explorer: container_name: appwrite-graphql-explorer diff --git a/docs/references/health/get-queue-usage-dump.md b/docs/references/health/get-queue-usage-dump.md deleted file mode 100644 index 3c95da1b8a..0000000000 --- a/docs/references/health/get-queue-usage-dump.md +++ /dev/null @@ -1 +0,0 @@ -Get the number of projects containing metrics that are waiting to be processed in the Appwrite internal queue server. \ No newline at end of file diff --git a/docs/references/health/get-queue-usage.md b/docs/references/health/get-queue-usage.md deleted file mode 100644 index 8e5b64e642..0000000000 --- a/docs/references/health/get-queue-usage.md +++ /dev/null @@ -1 +0,0 @@ -Get the number of metrics that are waiting to be processed in the Appwrite internal queue server. \ No newline at end of file diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 064fbcefa9..57300feb72 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -10,7 +10,6 @@ class Delete extends Event { protected string $type = ''; protected ?Document $document = null; - protected ?string $resourceType = null; protected ?string $resource = null; protected ?string $datetime = null; protected ?string $hourlyUsageRetentionDatetime = null; @@ -108,19 +107,6 @@ class Delete extends Event return $this; } - /** - * Sets the resource type for the delete event. - * - * @param string $resourceType - * @return self - */ - public function setResourceType(string $resourceType): self - { - $this->resourceType = $resourceType; - - return $this; - } - /** * Returns the set document for the delete event. * @@ -147,7 +133,6 @@ class Delete extends Event 'type' => $this->type, 'document' => $this->document, 'resource' => $this->resource, - 'resourceType' => $this->resourceType, 'datetime' => $this->datetime, 'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime ]); diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 05a28b418a..2c904973a9 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -270,7 +270,7 @@ class CalcTierStats extends Action $limit = $periods[$range]['limit']; $period = $periods[$range]['period']; - $requestDocs = $dbForProject->find('stats', [ + $requestDocs = $dbForProject->find('stats_v2', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), diff --git a/src/Appwrite/Platform/Tasks/CreateInfMetric.php b/src/Appwrite/Platform/Tasks/CreateInfMetric.php index 4b3f0e89fb..49b852ff6f 100644 --- a/src/Appwrite/Platform/Tasks/CreateInfMetric.php +++ b/src/Appwrite/Platform/Tasks/CreateInfMetric.php @@ -167,8 +167,8 @@ class CreateInfMetric extends Action try { $id = \md5("_inf_{$metric}"); - $dbForProject->deleteDocument('stats', $id); - $dbForProject->createDocument('stats', new Document([ + $dbForProject->deleteDocument('stats_v2', $id); + $dbForProject->createDocument('stats_v2', new Document([ '$id' => $id, 'metric' => $metric, 'period' => 'inf', @@ -190,7 +190,7 @@ class CreateInfMetric extends Action protected function getFromMetric(database $dbForProject, string $metric): int|float { - return $dbForProject->sum('stats', 'value', [ + return $dbForProject->sum('stats_v2', 'value', [ Query::equal('metric', [ $metric, ]), diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 37975979ce..96cb2d80cf 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -68,13 +68,18 @@ class Deletes extends Action $datetime = $payload['datetime'] ?? null; $hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null; $resource = $payload['resource'] ?? null; - $resourceType = $payload['resourceType'] ?? null; $document = new Document($payload['document'] ?? []); $project = new Document($payload['project'] ?? []); switch (strval($type)) { case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { + case DELETE_TYPE_DATABASES: + $this->deleteDatabase($getProjectDB, $document, $project); + break; + case DELETE_TYPE_COLLECTIONS: + $this->deleteCollection($getProjectDB, $document, $project); + break; case DELETE_TYPE_PROJECTS: $this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document); break; @@ -103,6 +108,10 @@ class Deletes extends Action $this->deleteRule($dbForConsole, $document); break; default: + if (\str_starts_with($document->getCollection(), 'database_')) { + $this->deleteCollection($getProjectDB, $document, $project); + break; + } Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); break; } @@ -136,7 +145,7 @@ class Deletes extends Action $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType); + $this->deleteCacheByResource($project, $getProjectDB, $resource); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate($project, $getProjectDB, $datetime); @@ -194,37 +203,32 @@ class Deletes extends Action * @param string $resource * @return void * @throws Authorization - * @param string|null $resourceType - * @throws Exception */ - private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void + private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); + $document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]); - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) - ); + if ($document) { + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); - $query[] = Query::equal('resource', [$resource]); - if (!empty($resourceType)) { - $query[] = Query::equal('resourceType', [$resourceType]); - } + $this->deleteById( + $document, + $dbForProject, + function ($document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); - $this->deleteByGroup( - 'cache', - $query, - $dbForProject, - function (Document $document) use ($cache, $projectId) { - $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); - - if ($cache->purge($document->getId())) { - Console::success('Deleting cache file: ' . $path); - } else { - Console::error('Failed to delete cache file: ' . $path); + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); + } } - } - ); + ); + } } /** @@ -264,6 +268,72 @@ class Deletes extends Action ); } + /** + * @param callable $getProjectDB + * @param Document $document + * @param Document $project + * @return void + * @throws Exception + */ + private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void + { + $databaseId = $document->getId(); + $dbForProject = $getProjectDB($project); + + $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) { + $this->deleteCollection($getProjectDB, $document, $project); + }); + + $dbForProject->deleteCollection('database_' . $document->getInternalId()); + $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project); + } + + /** + * @param callable $getProjectDB + * @param Document $document teams document + * @param Document $project + * @return void + * @throws Exception + */ + private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void + { + $collectionId = $document->getId(); + $collectionInternalId = $document->getInternalId(); + $databaseId = $document->getAttribute('databaseId'); + $databaseInternalId = $document->getAttribute('databaseInternalId'); + + $dbForProject = $getProjectDB($project); + + $relationships = \array_filter( + $document->getAttribute('attributes'), + fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP + ); + + foreach ($relationships as $relationship) { + if (!$relationship['twoWay']) { + continue; + } + $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); + $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); + $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); + $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); + } + + $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); + + $this->deleteByGroup('attributes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteByGroup('indexes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project); + } + /** * @param Database $dbForConsole * @param callable $getProjectDB @@ -275,7 +345,7 @@ class Deletes extends Action { $dbForProject = $getProjectDB($project); // Delete Usage stats - $this->deleteByGroup('stats', [ + $this->deleteByGroup('stats_v2', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']), ], $dbForProject); diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php index 6239f842e0..0fb705d0f7 100644 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -286,7 +286,7 @@ class Hamster extends Action $limit = $periodValue['limit']; $period = $periodValue['period']; - $requestDocs = $dbForProject->find('stats', [ + $requestDocs = $dbForProject->find('stats_v2', [ Query::equal('period', [$period]), Query::equal('metric', [$metric]), Query::limit($limit), diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 718dca8514..35c0e02cef 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -130,8 +130,8 @@ class Usage extends Action } break; case $document->getCollection() === 'databases': // databases - $collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS))); - $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS))); + $collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS))); + $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS))); if (!empty($collections['value'])) { $metrics[] = [ 'key' => METRIC_COLLECTIONS, @@ -149,7 +149,7 @@ class Usage extends Action case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; - $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS))); + $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS))); if (!empty($documents['value'])) { $metrics[] = [ @@ -164,8 +164,8 @@ class Usage extends Action break; case $document->getCollection() === 'buckets': - $files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES))); - $storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE))); + $files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES))); + $storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE))); if (!empty($files['value'])) { $metrics[] = [ @@ -183,13 +183,13 @@ class Usage extends Action break; case $document->getCollection() === 'functions': - $deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); - $deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); - $builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); - $buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); - $buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); - $executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); - $executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); + $deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); + $deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); + $builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); + $buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); + $buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); + $executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); + $executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); if (!empty($deployments['value'])) { $metrics[] = [ diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 1d296a929e..5a4324fd3e 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -78,7 +78,7 @@ class UsageDump extends Action $id = \md5("{$time}_{$period}_{$key}"); try { - $dbForProject->createDocument('stats', new Document([ + $dbForProject->createDocument('stats_v2', new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -89,14 +89,14 @@ class UsageDump extends Action } catch (Duplicate $th) { if ($value < 0) { $dbForProject->decreaseDocumentAttribute( - 'stats', + 'stats_v2', $id, 'value', abs($value) ); } else { $dbForProject->increaseDocumentAttribute( - 'stats', + 'stats_v2', $id, 'value', $value diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index ce86235634..863966eedd 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -295,7 +295,7 @@ class OpenAPI3 extends Format switch ((!empty($validator)) ? \get_class($validator) : '') { case 'Utopia\Validator\Text': $node['schema']['type'] = $validator->getType(); - $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; + $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; break; case 'Utopia\Validator\Boolean': $node['schema']['type'] = $validator->getType(); @@ -303,14 +303,14 @@ class OpenAPI3 extends Format break; case 'Utopia\Database\Validator\UID': $node['schema']['type'] = $validator->getType(); - $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; + $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; break; case 'Appwrite\Utopia\Database\Validator\CustomId': if ($route->getLabel('sdk.methodType', '') === 'upload') { $node['schema']['x-upload-id'] = true; } $node['schema']['type'] = $validator->getType(); - $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; + $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; break; case 'Utopia\Database\Validator\DatetimeValidator': $node['schema']['type'] = $validator->getType(); diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index a458c30aa6..c6ae7a7ff1 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -297,7 +297,7 @@ class Swagger2 extends Format switch ((!empty($validator)) ? \get_class($validator) : '') { case 'Utopia\Validator\Text': $node['type'] = $validator->getType(); - $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; + $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; break; case 'Utopia\Validator\Boolean': $node['type'] = $validator->getType(); @@ -308,11 +308,11 @@ class Swagger2 extends Format $node['x-upload-id'] = true; } $node['type'] = $validator->getType(); - $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; + $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; break; case 'Utopia\Database\Validator\UID': $node['type'] = $validator->getType(); - $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>'; + $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']'; break; case 'Utopia\Database\Validator\DatetimeValidator': $node['type'] = $validator->getType(); diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 091b3bb781..dde89a6a71 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -512,52 +512,4 @@ class HealthCustomServerTest extends Scope return []; } - - public function testUsageSuccess() - { - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertIsInt($response['body']['size']); - $this->assertLessThan(100, $response['body']['size']); - - /** - * Test for FAILURE - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage?threshold=0', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - $this->assertEquals(503, $response['headers']['status-code']); - } - - public function testUsageDumpSuccess() - { - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertIsInt($response['body']['size']); - $this->assertLessThan(100, $response['body']['size']); - - /** - * Test for FAILURE - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump?threshold=0', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - $this->assertEquals(503, $response['headers']['status-code']); - } }