diff --git a/Dockerfile b/Dockerfile index 88d5ed030b..2f1ea6f279 100755 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-stats-usage && \ - chmod +x /usr/local/bin/worker-stats-usage-dump && \ chmod +x /usr/local/bin/stats-resources && \ chmod +x /usr/local/bin/worker-stats-resources diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0c37e1a765..4b9d3d18f2 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -534,7 +534,6 @@ App::post('/v1/databases') } $queueForEvents->setParam('databaseId', $database->getId()); - $queueForStatsUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -830,9 +829,6 @@ App::delete('/v1/databases/:databaseId') ->setParam('databaseId', $database->getId()) ->setPayload($response->output($database, Response::MODEL_DATABASE)); - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation - $response->noContent(); }); @@ -2732,9 +2728,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->setContext('database', $db) ->setPayload($response->output($attribute, $model)); - $queueForStatsUsage - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->noContent(); }); @@ -3356,8 +3349,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); // per collection $response->addHeader('X-Debug-Operations', $operations); @@ -4141,8 +4133,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); // per collection $response->addHeader('X-Debug-Operations', 1); diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index e5336067c8..81a77733ff 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -756,38 +756,6 @@ App::get('/v1/health/queue/stats-usage') $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }); -App::get('/v1/health/queue/stats-usage-dump') - ->desc('Get usage dump queue') - ->groups(['api', 'health']) - ->label('scope', 'health.read') - ->label('sdk', new Method( - auth: [AuthType::KEY], - namespace: 'health', - name: 'getQueueStatsUsageDump', - description: '/docs/references/health/get-queue-stats-usage-dump.md', - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_HEALTH_QUEUE, - ) - ], - contentType: ContentType::JSON - )) - ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('publisher') - ->inject('response') - ->action(function (int|string $threshold, Publisher $publisher, Response $response) { - $threshold = \intval($threshold); - - $size = $publisher->getQueueSize(new Queue(Event::STATS_USAGE_DUMP_QUEUE_NAME)); - - if ($size >= $threshold) { - throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); - } - - $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); - }); - App::get('/v1/health/storage/local') ->desc('Get local storage') ->groups(['api', 'health']) @@ -954,7 +922,6 @@ App::get('/v1/health/queue/failed/:name') Event::FUNCTIONS_QUEUE_NAME, Event::STATS_RESOURCES_QUEUE_NAME, Event::STATS_USAGE_QUEUE_NAME, - Event::STATS_USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_QUEUE_NAME, Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index f34af1865e..25c8c30742 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -744,34 +744,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-worker-stats-usage-dump: - image: /: - entrypoint: worker-stats-usage-dump - <<: *x-logging - container_name: appwrite-worker-stats-usage-dump - restart: unless-stopped - networks: - - appwrite - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_USAGE_STATS - - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-task-scheduler-functions: image: /: entrypoint: schedule-functions diff --git a/app/worker.php b/app/worker.php index 6a51ee55be..232e0b3684 100644 --- a/app/worker.php +++ b/app/worker.php @@ -15,7 +15,6 @@ use Appwrite\Event\Messaging; use Appwrite\Event\Migration; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; -use Appwrite\Event\StatsUsageDump; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Executor\Executor; @@ -279,10 +278,6 @@ Server::setResource('queueForStatsUsage', function (Publisher $publisher) { return new StatsUsage($publisher); }, ['publisher']); -Server::setResource('queueForStatsUsageDump', function (Publisher $publisher) { - return new StatsUsageDump($publisher); -}, ['publisher']); - Server::setResource('queueForDatabase', function (Publisher $publisher) { return new EventDatabase($publisher); }, ['publisher']); diff --git a/bin/worker-stats-usage-dump b/bin/worker-stats-usage-dump deleted file mode 100644 index 98e3c2cac7..0000000000 --- a/bin/worker-stats-usage-dump +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php stats-usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3e01235f07..e8e51805d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -828,38 +828,6 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-stats-usage-dump: - entrypoint: worker-stats-usage-dump - <<: *x-logging - container_name: appwrite-worker-stats-usage-dump - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_USAGE_STATS - - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL - - _APP_DATABASE_SHARED_TABLES - - _APP_STATS_USAGE_DUAL_WRITING_DBS - appwrite-task-scheduler-functions: entrypoint: schedule-functions <<: *x-logging diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index a00cc45159..08faeea485 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -24,23 +24,12 @@ class Event public const FUNCTIONS_QUEUE_NAME = 'v1-functions'; public const FUNCTIONS_CLASS_NAME = 'FunctionsV1'; - /** remove */ - public const USAGE_QUEUE_NAME = 'v1-usage'; - public const USAGE_CLASS_NAME = 'UsageV1'; - - public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; - public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; - /** /remove */ - public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources'; public const STATS_RESOURCES_CLASS_NAME = 'StatsResourcesV1'; public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage'; public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1'; - public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump'; - public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1'; - public const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; public const WEBHOOK_CLASS_NAME = 'WebhooksV1'; diff --git a/src/Appwrite/Event/StatsUsageDump.php b/src/Appwrite/Event/StatsUsageDump.php deleted file mode 100644 index 0573a88040..0000000000 --- a/src/Appwrite/Event/StatsUsageDump.php +++ /dev/null @@ -1,44 +0,0 @@ -setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME) - ->setClass(Event::STATS_USAGE_DUMP_CLASS_NAME); - } - - /** - * Add Stats. - * - * @param array $stats - * @return self - */ - public function setStats(array $stats): self - { - $this->stats = $stats; - - return $this; - } - - /** - * Prepare the payload for the usage dump event. - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'stats' => $this->stats, - ]; - } -} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 4f4095aca4..eb544c140e 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -13,7 +13,6 @@ use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\StatsUsage; -use Appwrite\Platform\Workers\StatsUsageDump; use Appwrite\Platform\Workers\Webhooks; use Utopia\Platform\Service; @@ -32,7 +31,6 @@ class Workers extends Service ->addAction(Mails::getName(), new Mails()) ->addAction(Messaging::getName(), new Messaging()) ->addAction(Webhooks::getName(), new Webhooks()) - ->addAction(StatsUsageDump::getName(), new StatsUsageDump()) ->addAction(StatsUsage::getName(), new StatsUsage()) ->addAction(Migrations::getName(), new Migrations()) ->addAction(StatsResources::getName(), new StatsResources()) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index a755f723a0..66f285fcf5 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -2,17 +2,22 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\StatsUsageDump; use Exception; +use Throwable; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Registry\Registry; use Utopia\System\System; class StatsUsage extends Action { + /** + * In memory per project metrics calculation + */ private array $stats = []; private int $lastTriggeredTime = 0; private int $keys = 0; @@ -20,6 +25,77 @@ class StatsUsage extends Action private const BATCH_SIZE_DEVELOPMENT = 1; private const BATCH_SIZE_PRODUCTION = 10_000; + /** + * Stats for batch write separated per project + * @var array + */ + private array $projects = []; + + /** + * Array of stat documents to batch write to logsDB + * @var array + */ + private array $statDocuments = []; + + protected Registry $register; + + /** + * Metrics to skip writing to logsDB + * As these metrics are calculated separately + * by logs DB + * @var array + */ + protected array $skipBaseMetrics = [ + METRIC_DATABASES => true, + METRIC_BUCKETS => true, + METRIC_USERS => true, + METRIC_FUNCTIONS => true, + METRIC_TEAMS => true, + METRIC_MESSAGES => true, + METRIC_MAU => true, + METRIC_WEBHOOKS => true, + METRIC_PLATFORMS => true, + METRIC_PROVIDERS => true, + METRIC_TOPICS => true, + METRIC_KEYS => true, + METRIC_FILES => true, + METRIC_FILES_STORAGE => true, + METRIC_DEPLOYMENTS_STORAGE => true, + METRIC_BUILDS_STORAGE => true, + METRIC_DEPLOYMENTS => true, + METRIC_BUILDS => true, + METRIC_COLLECTIONS => true, + METRIC_DOCUMENTS => true, + METRIC_DATABASES_STORAGE => true, + ]; + + /** + * Skip metrics associated with parent IDs + * these need to be checked individually with `str_ends_with` + */ + protected array $skipParentIdMetrics = [ + '.files', + '.files.storage', + '.collections', + '.documents', + '.deployments', + '.deployments.storage', + '.builds', + '.builds.storage', + '.databases.storage' + ]; + + /** + * @var callable(): Database + */ + protected mixed $getLogsDB; + + protected array $periods = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + public static function getName(): string { return 'stats-usage'; @@ -41,7 +117,8 @@ class StatsUsage extends Action ->desc('Stats usage worker') ->inject('message') ->inject('getProjectDB') - ->inject('queueForStatsUsageDump') + ->inject('getLogsDB') + ->inject('register') ->callback([$this, 'action']); $this->lastTriggeredTime = time(); @@ -49,14 +126,17 @@ class StatsUsage extends Action /** * @param Message $message - * @param callable $getProjectDB - * @param StatsUsageDump $queueForStatsUsageDump + * @param callable(): Database $getProjectDB + * @param callable(): Database $getLogsDB + * @param Registry $register * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump): void + public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void { + $this->getLogsDB = $getLogsDB; + $this->register = $register; $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); @@ -98,9 +178,7 @@ class StatsUsage extends Action ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); - $queueForStatsUsageDump - ->setStats($this->stats) - ->trigger(); + $this->commitToDB($getProjectDB); $this->stats = []; $this->keys = 0; @@ -114,7 +192,7 @@ class StatsUsage extends Action * @param Document $project * @param Document $document * @param array $metrics - * @param callable $getProjectDB + * @param callable(): Database $getProjectDB * @return void */ private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void @@ -246,8 +324,128 @@ class StatsUsage extends Action default: break; } - } catch (\Throwable $e) { + } catch (Throwable $e) { console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } + + /** + * Commit stats to DB + * @param callable(): Database $getProjectDB + * @return void + */ + public function commitToDb(callable $getProjectDB): void + { + foreach ($this->stats as $stats) { + $project = $stats['project'] ?? new Document([]); + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + $receivedAt = $stats['receivedAt'] ?? null; + if ($numberOfKeys === 0) { + continue; + } + + console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + + try { + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + foreach ($this->periods as $period => $format) { + $time = null; + + if ($period !== 'inf') { + $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + } + $id = \md5("{$time}_{$period}_{$key}"); + + $document = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + + + $this->projects[$project->getInternalId()]['project'] = new Document([ + '$id' => $project->getId(), + '$internalId' => $project->getInternalId(), + 'database' => $project->getAttribute('database'), + ]); + $this->projects[$project->getInternalId()]['stats'][] = $document; + + $this->prepareForLogsDB($project, $document); + } + } + } catch (Exception $e) { + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); + } + } + + foreach ($this->projects as $internalId => $projectStats) { + if (empty($internalId)) { + continue; + } + try { + $dbForProject = $getProjectDB($projectStats['project']); + Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); + $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); + Console::success('Batch successfully written to DB'); + + unset($this->projects[$internalId]); + } catch (Throwable $e) { + Console::error('Error processing stats: ' . $e->getMessage()); + } + } + + $this->writeToLogsDB(); + + } + + protected function prepareForLogsDB(Document $project, Document $stat) + { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { + return; + } + if (array_key_exists($stat->getAttribute('metric'), $this->skipBaseMetrics)) { + return; + } + foreach ($this->skipParentIdMetrics as $skipMetric) { + if (str_ends_with($stat->getAttribute('metric'), $skipMetric)) { + return; + } + } + $documentClone = clone $stat; + $documentClone->setAttribute('$tenant', (int) $project->getInternalId()); + $this->statDocuments[] = $documentClone; + } + + protected function writeToLogsDB(): void + { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { + Console::log('Dual Writing is disabled. Skipping...'); + return; + } + + $dbForLogs = call_user_func($this->getLogsDB); + $dbForLogs + ->setTenant(null) + ->setTenantPerDocument(true); + + try { + Console::log('Processing batch with ' . count($this->statDocuments) . ' stats'); + $dbForLogs->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + $this->statDocuments + ); + Console::success('Usage logs pushed to Logs DB'); + } catch (Throwable $th) { + Console::error($th->getMessage()); + } + $this->register->get('pools')->get('logs')->reclaim(); + } } diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 119a9e7288..b9d486e0d8 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -4,7 +4,6 @@ namespace Appwrite\Platform\Workers; use Appwrite\Extend\Exception; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; @@ -12,6 +11,9 @@ use Utopia\Queue\Message; use Utopia\Registry\Registry; use Utopia\System\System; +/** + * TODO remove later + */ class StatsUsageDump extends Action { public const METRIC_COLLECTION_LEVEL_STORAGE = 4; @@ -135,11 +137,6 @@ class StatsUsageDump extends Action } if (str_contains($key, METRIC_DATABASES_STORAGE)) { - try { - $this->handleDatabaseStorage($key, $dbForProject, $project, $receivedAt); - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); - } continue; } @@ -160,7 +157,7 @@ class StatsUsageDump extends Action 'region' => System::getEnv('_APP_REGION', 'default'), ]); - $documentClone = new Document($document->getArrayCopy()); + $documentClone = clone $document; $dbForProject->createOrUpdateDocumentsWithIncrease( 'stats', @@ -177,157 +174,6 @@ class StatsUsageDump extends Action } } - private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project, string $receivedAt): void - { - $data = explode('.', $key); - $start = microtime(true); - - $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) use ($receivedAt) { - $id = \md5("{$time}_{$period}_{$key}"); - - $document = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - $documentClone = new Document($document->getArrayCopy()); - $dbForProject->createOrUpdateDocumentsWithIncrease( - 'stats', - 'value', - [$document] - ); - $this->writeToLogsDB($project, $documentClone); - }; - - foreach ($this->periods as $period => $format) { - $time = null; - - if ($period !== 'inf') { - $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); - } - $id = \md5("{$time}_{$period}_{$key}"); - - $value = 0; - $previousValue = 0; - try { - $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); - } catch (\Exception $e) { - // No previous value - } - - switch (count($data)) { - // Collection Level - case self::METRIC_COLLECTION_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; - - try { - $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - // Compare with previous value - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Collection - $updateMetric($dbForProject, $project, $diff, $key, $period, $time); - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - // Database Level - case self::METRIC_DATABASE_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - - $collections = []; - try { - $collections = $dbForProject->find('database_' . $databaseInternalId); - } catch (\Exception $e) { - // Database not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - // Project Level - case self::METRIC_PROJECT_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); - // Get all project databases - $databases = $dbForProject->find('database'); - - // Recalculate all databases - foreach ($databases as $database) { - $collections = $dbForProject->find('database_' . $database->getInternalId()); - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - } - - $diff = $value - $previousValue; - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - } - } - - $end = microtime(true); - - console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); - } - protected function writeToLogsDB(Document $project, Document $document): void { if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index d0e37e5639..4473cab30c 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -6,7 +6,6 @@ use Appwrite\Functions\Specification; use Appwrite\Tests\Retry; use CURLFile; use DateTime; -use PHPUnit\Framework\ExpectationFailedException; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -593,255 +592,6 @@ class UsageTest extends Scope return $data; } - public function testDatabaseStoragePrepare(): array - { - $response = $this->client->call( - Client::METHOD_POST, - '/databases', - array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), - [ - 'databaseId' => 'unique()', - 'name' => 'dbStorageStats', - ] - ); - - $this->assertNotEmpty($response['body']['$id']); - $databaseId = $response['body']['$id']; - - $response = $this->client->call( - Client::METHOD_POST, - '/databases/' . $databaseId . '/collections', - array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), - [ - 'collectionId' => 'unique()', - 'name' => 'collectionStorageStats', - 'documentSecurity' => false, - 'permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ] - ); - - $this->assertNotEmpty($response['body']['$id']); - $collectionId = $response['body']['$id']; - - $response = $this->client->call( - Client::METHOD_POST, - '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', - array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), - [ - 'key' => 'data', - 'size' => 100000, - 'required' => true, - ] - ); - - return [ - 'databaseId' => $databaseId, - 'collectionId' => $collectionId, - ]; - } - - // /** @depends testDatabaseStoragePrepare */ - // #[Retry(count: 1)] - // public function testDatabaseStorageStatsCreateDocument(array $data): array - // { - // $databaseId = $data['databaseId']; - // $collectionId = $data['collectionId']; - - // $originalProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $originalProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $originalProjectMetrics['body']); - - // $originalProjectMetrics = $originalProjectMetrics['body']; - - // $originalDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $originalDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $originalDatabaseMetrics['body']); - // $originalDatabaseMetrics = $originalDatabaseMetrics['body']; - - // // Create documents - // for ($i = 0; $i < 100; $i++) { - // $response = $this->client->call( - // Client::METHOD_POST, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', - // array_merge([ - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'documentId' => 'unique()', - // 'data' => ['data' => str_repeat('a', 10000)], - // ] - // ); - - // $this->assertEquals(201, $response['headers']['status-code']); - // } - - // for ($i = 0; $i < 3; $i++) { - // try { - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); - // $this->assertGreaterThan($originalProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); - - // $newProjectMetrics = $newProjectMetrics['body']; - - // $newDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); - // $this->assertGreaterThan($originalDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); - - // $newDatabaseMetrics = $newDatabaseMetrics['body']; - - // return [ - // 'databaseId' => $databaseId, - // 'collectionId' => $collectionId, - // 'currentProjectMetrics' => $newProjectMetrics, - // 'currentDatabaseMetrics' => $newDatabaseMetrics, - // ]; - // } catch (ExpectationFailedException $e) { - // if ($i === 2) { - // throw $e; - // } - // continue; - // } - // } - // } - - // /** @depends testDatabaseStorageStatsCreateDocument */ - // #[Retry(count: 1)] - // public function testDatabaseStorageStatsDeleteDocument(array $data): array - // { - // $databaseId = $data['databaseId']; - // $collectionId = $data['collectionId']; - // $currentProjectMetrics = $data['currentProjectMetrics']; - // $currentDatabaseMetrics = $data['currentDatabaseMetrics']; - - // $documents = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', - // array_merge([ - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'queries' => [ - // Query::limit(50)->toString() - // ] - // ] - // ); - - // foreach ($documents['body']['documents'] as $document) { - // $response = $this->client->call( - // Client::METHOD_DELETE, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document['$id'], - // array_merge([ - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()) - // ); - - // $this->assertEquals(204, $response['headers']['status-code']); - // } - - // for ($i = 0; $i < 3; $i++) { - // try { - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); - // $this->assertLessThan($currentProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); - - // $newProjectMetrics = $newProjectMetrics['body']; - - // $newDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); - // $this->assertLessThan($currentDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); - - // $newDatabaseMetrics = $newDatabaseMetrics['body']; - - // return [ - // 'databaseId' => $databaseId, - // 'collectionId' => $collectionId, - // 'currentProjectMetrics' => $newProjectMetrics, - // 'currentDatabaseMetrics' => $newDatabaseMetrics, - // ]; - // } catch (ExpectationFailedException $e) { - // if ($i === 2) { - // throw $e; - // } - // continue; - // } - // } - - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - // } - /** @depends testDatabaseStats */ public function testPrepareFunctionsStats(array $data): array { diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 04b1408cd0..4b7062dc22 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -541,28 +541,4 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(503, $response['headers']['status-code']); } - - public function testStatsUsageDumpSuccess() - { - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertIsInt($response['body']['size']); - $this->assertLessThan(100, $response['body']['size']); - - /** - * Test for FAILURE - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump?threshold=0', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - $this->assertEquals(503, $response['headers']['status-code']); - } }