From 6ea4098679cffb3dd972b67362da377763e01e32 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 8 Aug 2022 01:33:34 +0000 Subject: [PATCH] new metrics aggregation --- composer.lock | 2 +- src/Appwrite/Stats/Usage.php | 30 ++--- src/Appwrite/Stats/UsageDB.php | 230 ++++++++++++++++++++++++++++++--- 3 files changed, 226 insertions(+), 36 deletions(-) diff --git a/composer.lock b/composer.lock index 93b8455cac..bce5b90d88 100644 --- a/composer.lock +++ b/composer.lock @@ -5370,5 +5370,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php index 529a2bf67d..42eeb0b11f 100644 --- a/src/Appwrite/Stats/Usage.php +++ b/src/Appwrite/Stats/Usage.php @@ -188,17 +188,9 @@ class Usage ], ]; - protected array $periods = [ - [ - 'key' => '30m', - 'multiplier' => 1800, - 'startTime' => '-24 hours', - ], - [ - 'key' => '1d', - 'multiplier' => 86400, - 'startTime' => '-90 days', - ], + protected array $period = [ + 'key' => '30m', + 'startTime' => '-24 hours', ]; public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) @@ -342,15 +334,13 @@ class Usage public function collect(): void { foreach ($this->metrics as $metric => $options) { //for each metrics - foreach ($this->periods as $period) { // aggregate data for each period - try { - $this->syncFromInfluxDB($metric, $options, $period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; - } + try { + $this->syncFromInfluxDB($metric, $options, $this->period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; } } } diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php index 44f9e92bb0..10376bc377 100644 --- a/src/Appwrite/Stats/UsageDB.php +++ b/src/Appwrite/Stats/UsageDB.php @@ -4,14 +4,45 @@ namespace Appwrite\Stats; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; class UsageDB extends Usage { + protected array $periods = [ + [ + 'key' => '30m', + 'multiplier' => 1800, + ], + [ + 'key' => '1d', + 'multiplier' => 86400, + ], + ]; + public function __construct(Database $database, callable $errorHandler = null) { $this->database = $database; $this->errorHandler = $errorHandler; } + /** + * Create Per Period Metric + * Create given metric for each defined period + * + * @param string $projectId + * @param string $metric + * @param int $value + * + * @return void + */ + private function createPerPeriodMetric(string $projectId, string $metric, int $value): void + { + foreach ($this->periods as $options) { + $period = $options['key']; + $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); + $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); + } + } + /** * Create or Update Mertic * Create or update each metric in the stats collection for the given project @@ -22,11 +53,8 @@ class UsageDB extends Usage * * @return void */ - private function createOrUpdateMetric(string $projectId, string $metric, int $value): void + private function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void { - foreach ($this->periods as $options) { - $period = $options['key']; - $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); $id = \md5("{$time}_{$period}_{$metric}"); $this->database->setNamespace('_' . $projectId); @@ -55,7 +83,6 @@ class UsageDB extends Usage throw $e; } } - } } /** @@ -114,13 +141,13 @@ class UsageDB extends Usage * * @return int */ - private function sum(string $projectId, string $collection, string $attribute, string $metric): int + private function sum(string $projectId, string $collection, string $attribute, string $metric, array $queries = []): int { $this->database->setNamespace('_' . $projectId); try { - $sum = (int) $this->database->sum($collection, $attribute); - $this->createOrUpdateMetric($projectId, $metric, $sum); + $sum = (int) $this->database->sum($collection, $attribute, $queries); + $this->createPerPeriodMetric($projectId, $metric, $sum); return $sum; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { @@ -147,7 +174,7 @@ class UsageDB extends Usage try { $count = $this->database->count($collection); - $this->createOrUpdateMetric($projectId, $metric, $count); + $this->createPerPeriodMetric($projectId, $metric, $count); return $count; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { @@ -214,10 +241,10 @@ class UsageDB extends Usage $projectFilesTotal += $sum; }); - $this->createOrUpdateMetric($projectId, 'storage.files.count', $projectFilesCount); - $this->createOrUpdateMetric($projectId, 'storage.files.total', $projectFilesTotal); + $this->createPerPeriodMetric($projectId, 'storage.files.count', $projectFilesCount); + $this->createPerPeriodMetric($projectId, 'storage.files.total', $projectFilesTotal); - $this->createOrUpdateMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); + $this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); } /** @@ -251,11 +278,176 @@ class UsageDB extends Usage $databaseDocumentsCount += $count; }); - $this->createOrUpdateMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount); + $this->createPerPeriodMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount); }); - $this->createOrUpdateMetric($projectId, 'databases.collections.count', $projectCollectionsCount); - $this->createOrUpdateMetric($projectId, 'databases.documents.count', $projectDocumentsCount); + $this->createPerPeriodMetric($projectId, 'databases.collections.count', $projectCollectionsCount); + $this->createPerPeriodMetric($projectId, 'databases.documents.count', $projectDocumentsCount); + } + + protected function aggregateDatabaseMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $databasesGeneralMetrics = [ + 'databases.create', + 'databases.read', + 'databases.update', + 'databases.delete', + 'databases.collections.create', + 'databases.collections.read', + 'databases.collections.update', + 'databases.collections.delete', + 'databases.documents.create', + 'databases.documents.read', + 'databases.documents.update', + 'databases.documents.delete', + ]; + + foreach($databasesGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesDatabaseMetrics = [ + 'databases.databaseId.collections.create', + 'databases.databaseId.collections.read', + 'databases.databaseId.collections.update', + 'databases.databaseId.collections.delete', + 'databases.databaseId.documents.create', + 'databases.databaseId.documents.read', + 'databases.databaseId.documents.update', + 'databases.databaseId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'databases', [], function(Document $database) use ($databasesDatabaseMetrics, $projectId) { + $databaseId = $database->getId(); + foreach ($databasesDatabaseMetrics as $metric) { + $metric = str_replace('databaseId', $databaseId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesCollectionMetrics = [ + 'databases.' . $databaseId . '.collections.collectionId.documents.create', + 'databases.' . $databaseId . '.collections.collectionId.documents.read', + 'databases.' . $databaseId . '.collections.collectionId.documents.update', + 'databases.' . $databaseId . '.collections.collectionId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function(Document $collection) use ($databasesCollectionMetrics, $projectId) { + $collectionId = $collection->getId(); + foreach ($databasesCollectionMetrics as $metric) { + $metric = str_replace('collectionId', $collectionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + $this->aggregateMonthlyMetric($projectId, $metric); + } + }); + }); + } + + protected function aggregateStorageMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $storageGeneralMetrics = [ + 'storage.buckets.create', + 'storage.buckets.read', + 'storage.buckets.update', + 'storage.buckets.delete', + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach($storageGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $storageBucketMetrics = [ + 'storage.buckets.bucketId.files.create', + 'storage.buckets.bucketId.files.read', + 'storage.buckets.bucketId.files.update', + 'storage.buckets.bucketId.files.delete', + ]; + + $this->foreachDocument($projectId, 'buckets', [], function(Document $bucket) use ($storageBucketMetrics, $projectId) { + $bucketId = $bucket->getId(); + foreach ($storageBucketMetrics as $metric) { + $metric = str_replace('bucketId', $bucketId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateFunctionMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $this->aggregateDailyMetric($projectId, 'executions'); + + $functionMetrics = [ + 'functions.functionId.executions', + 'functions.functionId.compute', + 'function.functionId.failure', + ]; + + $this->foreachDocument($projectId, 'functions', [], function(Document $function) use ($functionMetrics, $projectId) { + $functionId = $function->getId(); + foreach ($functionMetrics as $metric) { + $metric = str_replace('functionId', $functionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateUsersMetrics(string $projectId): void + { + $metrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + 'users.sessions.create', + 'users.sessions.delete' + ]; + + foreach($metrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + } + + protected function aggregateGeneralMetrics(string $projectId): void + { + $this->aggregateDailyMetric($projectId, 'requests'); + $this->aggregateDailyMetric($projectId, 'network'); + } + + protected function aggregateDailyMetric(string $projectId, string $metric): void + { + $beginOfDay = strtotime("today"); + $endOfDay = strtotime("tomorrow", $beginOfDay) - 1; + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); + } + + protected function aggregateMonthlyMetric(string $projectId, string $metric): void + { + $beginOfMonth = strtotime("first day of the month"); + $endOfMonth = strtotime("last day of the month"); + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['1d']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); } /** @@ -272,6 +464,14 @@ class UsageDB extends Usage $this->usersStats($projectId); $this->databaseStats($projectId); $this->storageStats($projectId); + + // Aggregate new metrics from already collected usage metrics + // for lower time period (1day and 1 month metric from 30 minute metrics) + $this->aggregateGeneralMetrics($projectId); + $this->aggregateFunctionMetrics($projectId); + $this->aggregateDatabaseMetrics($projectId); + $this->aggregateStorageMetrics($projectId); + $this->aggregateUsersMetrics($projectId); }); } }