From 6ea4098679cffb3dd972b67362da377763e01e32 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 8 Aug 2022 01:33:34 +0000 Subject: [PATCH 1/3] new metrics aggregation --- composer.lock | 2 +- src/Appwrite/Stats/Usage.php | 30 ++--- src/Appwrite/Stats/UsageDB.php | 230 ++++++++++++++++++++++++++++++--- 3 files changed, 226 insertions(+), 36 deletions(-) diff --git a/composer.lock b/composer.lock index 93b8455cac..bce5b90d88 100644 --- a/composer.lock +++ b/composer.lock @@ -5370,5 +5370,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php index 529a2bf67d..42eeb0b11f 100644 --- a/src/Appwrite/Stats/Usage.php +++ b/src/Appwrite/Stats/Usage.php @@ -188,17 +188,9 @@ class Usage ], ]; - protected array $periods = [ - [ - 'key' => '30m', - 'multiplier' => 1800, - 'startTime' => '-24 hours', - ], - [ - 'key' => '1d', - 'multiplier' => 86400, - 'startTime' => '-90 days', - ], + protected array $period = [ + 'key' => '30m', + 'startTime' => '-24 hours', ]; public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) @@ -342,15 +334,13 @@ class Usage public function collect(): void { foreach ($this->metrics as $metric => $options) { //for each metrics - foreach ($this->periods as $period) { // aggregate data for each period - try { - $this->syncFromInfluxDB($metric, $options, $period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; - } + try { + $this->syncFromInfluxDB($metric, $options, $this->period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; } } } diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php index 44f9e92bb0..10376bc377 100644 --- a/src/Appwrite/Stats/UsageDB.php +++ b/src/Appwrite/Stats/UsageDB.php @@ -4,14 +4,45 @@ namespace Appwrite\Stats; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; class UsageDB extends Usage { + protected array $periods = [ + [ + 'key' => '30m', + 'multiplier' => 1800, + ], + [ + 'key' => '1d', + 'multiplier' => 86400, + ], + ]; + public function __construct(Database $database, callable $errorHandler = null) { $this->database = $database; $this->errorHandler = $errorHandler; } + /** + * Create Per Period Metric + * Create given metric for each defined period + * + * @param string $projectId + * @param string $metric + * @param int $value + * + * @return void + */ + private function createPerPeriodMetric(string $projectId, string $metric, int $value): void + { + foreach ($this->periods as $options) { + $period = $options['key']; + $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); + $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); + } + } + /** * Create or Update Mertic * Create or update each metric in the stats collection for the given project @@ -22,11 +53,8 @@ class UsageDB extends Usage * * @return void */ - private function createOrUpdateMetric(string $projectId, string $metric, int $value): void + private function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void { - foreach ($this->periods as $options) { - $period = $options['key']; - $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); $id = \md5("{$time}_{$period}_{$metric}"); $this->database->setNamespace('_' . $projectId); @@ -55,7 +83,6 @@ class UsageDB extends Usage throw $e; } } - } } /** @@ -114,13 +141,13 @@ class UsageDB extends Usage * * @return int */ - private function sum(string $projectId, string $collection, string $attribute, string $metric): int + private function sum(string $projectId, string $collection, string $attribute, string $metric, array $queries = []): int { $this->database->setNamespace('_' . $projectId); try { - $sum = (int) $this->database->sum($collection, $attribute); - $this->createOrUpdateMetric($projectId, $metric, $sum); + $sum = (int) $this->database->sum($collection, $attribute, $queries); + $this->createPerPeriodMetric($projectId, $metric, $sum); return $sum; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { @@ -147,7 +174,7 @@ class UsageDB extends Usage try { $count = $this->database->count($collection); - $this->createOrUpdateMetric($projectId, $metric, $count); + $this->createPerPeriodMetric($projectId, $metric, $count); return $count; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { @@ -214,10 +241,10 @@ class UsageDB extends Usage $projectFilesTotal += $sum; }); - $this->createOrUpdateMetric($projectId, 'storage.files.count', $projectFilesCount); - $this->createOrUpdateMetric($projectId, 'storage.files.total', $projectFilesTotal); + $this->createPerPeriodMetric($projectId, 'storage.files.count', $projectFilesCount); + $this->createPerPeriodMetric($projectId, 'storage.files.total', $projectFilesTotal); - $this->createOrUpdateMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); + $this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); } /** @@ -251,11 +278,176 @@ class UsageDB extends Usage $databaseDocumentsCount += $count; }); - $this->createOrUpdateMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount); + $this->createPerPeriodMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount); }); - $this->createOrUpdateMetric($projectId, 'databases.collections.count', $projectCollectionsCount); - $this->createOrUpdateMetric($projectId, 'databases.documents.count', $projectDocumentsCount); + $this->createPerPeriodMetric($projectId, 'databases.collections.count', $projectCollectionsCount); + $this->createPerPeriodMetric($projectId, 'databases.documents.count', $projectDocumentsCount); + } + + protected function aggregateDatabaseMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $databasesGeneralMetrics = [ + 'databases.create', + 'databases.read', + 'databases.update', + 'databases.delete', + 'databases.collections.create', + 'databases.collections.read', + 'databases.collections.update', + 'databases.collections.delete', + 'databases.documents.create', + 'databases.documents.read', + 'databases.documents.update', + 'databases.documents.delete', + ]; + + foreach($databasesGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesDatabaseMetrics = [ + 'databases.databaseId.collections.create', + 'databases.databaseId.collections.read', + 'databases.databaseId.collections.update', + 'databases.databaseId.collections.delete', + 'databases.databaseId.documents.create', + 'databases.databaseId.documents.read', + 'databases.databaseId.documents.update', + 'databases.databaseId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'databases', [], function(Document $database) use ($databasesDatabaseMetrics, $projectId) { + $databaseId = $database->getId(); + foreach ($databasesDatabaseMetrics as $metric) { + $metric = str_replace('databaseId', $databaseId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesCollectionMetrics = [ + 'databases.' . $databaseId . '.collections.collectionId.documents.create', + 'databases.' . $databaseId . '.collections.collectionId.documents.read', + 'databases.' . $databaseId . '.collections.collectionId.documents.update', + 'databases.' . $databaseId . '.collections.collectionId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function(Document $collection) use ($databasesCollectionMetrics, $projectId) { + $collectionId = $collection->getId(); + foreach ($databasesCollectionMetrics as $metric) { + $metric = str_replace('collectionId', $collectionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + $this->aggregateMonthlyMetric($projectId, $metric); + } + }); + }); + } + + protected function aggregateStorageMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $storageGeneralMetrics = [ + 'storage.buckets.create', + 'storage.buckets.read', + 'storage.buckets.update', + 'storage.buckets.delete', + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach($storageGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $storageBucketMetrics = [ + 'storage.buckets.bucketId.files.create', + 'storage.buckets.bucketId.files.read', + 'storage.buckets.bucketId.files.update', + 'storage.buckets.bucketId.files.delete', + ]; + + $this->foreachDocument($projectId, 'buckets', [], function(Document $bucket) use ($storageBucketMetrics, $projectId) { + $bucketId = $bucket->getId(); + foreach ($storageBucketMetrics as $metric) { + $metric = str_replace('bucketId', $bucketId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateFunctionMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $this->aggregateDailyMetric($projectId, 'executions'); + + $functionMetrics = [ + 'functions.functionId.executions', + 'functions.functionId.compute', + 'function.functionId.failure', + ]; + + $this->foreachDocument($projectId, 'functions', [], function(Document $function) use ($functionMetrics, $projectId) { + $functionId = $function->getId(); + foreach ($functionMetrics as $metric) { + $metric = str_replace('functionId', $functionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateUsersMetrics(string $projectId): void + { + $metrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + 'users.sessions.create', + 'users.sessions.delete' + ]; + + foreach($metrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + } + + protected function aggregateGeneralMetrics(string $projectId): void + { + $this->aggregateDailyMetric($projectId, 'requests'); + $this->aggregateDailyMetric($projectId, 'network'); + } + + protected function aggregateDailyMetric(string $projectId, string $metric): void + { + $beginOfDay = strtotime("today"); + $endOfDay = strtotime("tomorrow", $beginOfDay) - 1; + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); + } + + protected function aggregateMonthlyMetric(string $projectId, string $metric): void + { + $beginOfMonth = strtotime("first day of the month"); + $endOfMonth = strtotime("last day of the month"); + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['1d']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); } /** @@ -272,6 +464,14 @@ class UsageDB extends Usage $this->usersStats($projectId); $this->databaseStats($projectId); $this->storageStats($projectId); + + // Aggregate new metrics from already collected usage metrics + // for lower time period (1day and 1 month metric from 30 minute metrics) + $this->aggregateGeneralMetrics($projectId); + $this->aggregateFunctionMetrics($projectId); + $this->aggregateDatabaseMetrics($projectId); + $this->aggregateStorageMetrics($projectId); + $this->aggregateUsersMetrics($projectId); }); } } From 98330cdfa11ddec53dfe3e09b351a0cc8f093f8a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 8 Aug 2022 02:20:54 +0000 Subject: [PATCH 2/3] fixes --- src/Appwrite/Stats/UsageDB.php | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php index 10376bc377..799cdfc832 100644 --- a/src/Appwrite/Stats/UsageDB.php +++ b/src/Appwrite/Stats/UsageDB.php @@ -141,12 +141,13 @@ class UsageDB extends Usage * * @return int */ - private function sum(string $projectId, string $collection, string $attribute, string $metric, array $queries = []): int + private function sum(string $projectId, string $collection, string $attribute, string $metric, int $multiplier = 1): int { $this->database->setNamespace('_' . $projectId); try { - $sum = (int) $this->database->sum($collection, $attribute, $queries); + $sum = $this->database->sum($collection, $attribute); + $sum = (int) ($sum * $multiplier); $this->createPerPeriodMetric($projectId, $metric, $sum); return $sum; } catch (\Exception $e) { @@ -156,6 +157,7 @@ class UsageDB extends Usage throw $e; } } + return 0; } /** @@ -183,6 +185,7 @@ class UsageDB extends Usage throw $e; } } + return 0; } /** @@ -247,6 +250,22 @@ class UsageDB extends Usage $this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); } + /** + * Compute Stats + * Metrics: functions.executionTime, functions.buildTime, functions.compute, + * + * @param string $projectId + * + * @return void + */ + private function computeStats(string $projectId): void + { + $executionTotal = $this->sum($projectId, 'executions', 'time', 'functions.executionTime', 1000); // in ms + $buildTotal = $this->sum($projectId, 'builds', 'duration', 'functions.buildTime', 1000); // in ms + + $this->createPerPeriodMetric($projectId, 'functions.compute', $executionTotal + $buildTotal); //in ms + } + /** * Database Stats * Collect all database stats @@ -464,6 +483,7 @@ class UsageDB extends Usage $this->usersStats($projectId); $this->databaseStats($projectId); $this->storageStats($projectId); + $this->computeStats($projectId); // Aggregate new metrics from already collected usage metrics // for lower time period (1day and 1 month metric from 30 minute metrics) From 84a9dcf6cb607c4d4975ea8e2e14ca66be728aea Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 8 Aug 2022 06:59:52 +0000 Subject: [PATCH 3/3] formatting --- src/Appwrite/Stats/Usage.php | 3 -- src/Appwrite/Stats/UsageDB.php | 89 ++++++++++++++++++++-------------- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php index 8fbe9a49ad..560bd0ff0f 100644 --- a/src/Appwrite/Stats/Usage.php +++ b/src/Appwrite/Stats/Usage.php @@ -28,9 +28,6 @@ class Usage 'outbound' => [ 'table' => 'appwrite_usage_network_outbound', ], - 'executions' => [ - 'table' => 'appwrite_usage_executions_all', - ], 'databases.create' => [ 'table' => 'appwrite_usage_databases_create', ], diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php index 799cdfc832..1caef7acdb 100644 --- a/src/Appwrite/Stats/UsageDB.php +++ b/src/Appwrite/Stats/UsageDB.php @@ -34,13 +34,19 @@ class UsageDB extends Usage * * @return void */ - private function createPerPeriodMetric(string $projectId, string $metric, int $value): void + private function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void { foreach ($this->periods as $options) { $period = $options['key']; $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); } + + // Required for billing + if ($monthly) { + $time = strtotime("first day of the month"); + $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value); + } } /** @@ -58,31 +64,31 @@ class UsageDB extends Usage $id = \md5("{$time}_{$period}_{$metric}"); $this->database->setNamespace('_' . $projectId); - try { - $document = $this->database->getDocument('stats', $id); - if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 1, - ])); - } else { - $this->database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - } catch (\Exception$e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } + try { + $document = $this->database->getDocument('stats', $id); + if ($document->isEmpty()) { + $this->database->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 1, + ])); + } else { + $this->database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); } + } catch (\Exception$e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } + } } /** @@ -307,7 +313,7 @@ class UsageDB extends Usage protected function aggregateDatabaseMetrics(string $projectId): void { $this->database->setNamespace('_' . $projectId); - + $databasesGeneralMetrics = [ 'databases.create', 'databases.read', @@ -323,7 +329,7 @@ class UsageDB extends Usage 'databases.documents.delete', ]; - foreach($databasesGeneralMetrics as $metric) { + foreach ($databasesGeneralMetrics as $metric) { $this->aggregateDailyMetric($projectId, $metric); } @@ -338,7 +344,7 @@ class UsageDB extends Usage 'databases.databaseId.documents.delete', ]; - $this->foreachDocument($projectId, 'databases', [], function(Document $database) use ($databasesDatabaseMetrics, $projectId) { + $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) { $databaseId = $database->getId(); foreach ($databasesDatabaseMetrics as $metric) { $metric = str_replace('databaseId', $databaseId, $metric); @@ -352,12 +358,11 @@ class UsageDB extends Usage 'databases.' . $databaseId . '.collections.collectionId.documents.delete', ]; - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function(Document $collection) use ($databasesCollectionMetrics, $projectId) { + $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) { $collectionId = $collection->getId(); foreach ($databasesCollectionMetrics as $metric) { $metric = str_replace('collectionId', $collectionId, $metric); $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); } }); }); @@ -366,7 +371,7 @@ class UsageDB extends Usage protected function aggregateStorageMetrics(string $projectId): void { $this->database->setNamespace('_' . $projectId); - + $storageGeneralMetrics = [ 'storage.buckets.create', 'storage.buckets.read', @@ -378,7 +383,7 @@ class UsageDB extends Usage 'storage.files.delete', ]; - foreach($storageGeneralMetrics as $metric) { + foreach ($storageGeneralMetrics as $metric) { $this->aggregateDailyMetric($projectId, $metric); } @@ -389,7 +394,7 @@ class UsageDB extends Usage 'storage.buckets.bucketId.files.delete', ]; - $this->foreachDocument($projectId, 'buckets', [], function(Document $bucket) use ($storageBucketMetrics, $projectId) { + $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) { $bucketId = $bucket->getId(); foreach ($storageBucketMetrics as $metric) { $metric = str_replace('bucketId', $bucketId, $metric); @@ -402,15 +407,19 @@ class UsageDB extends Usage { $this->database->setNamespace('_' . $projectId); - $this->aggregateDailyMetric($projectId, 'executions'); + $this->aggregateDailyMetric($projectId, 'functions.executions'); + $this->aggregateDailyMetric($projectId, 'functions.builds'); + $this->aggregateDailyMetric($projectId, 'functions.failures'); $functionMetrics = [ 'functions.functionId.executions', + 'functions.functionId.builds', 'functions.functionId.compute', - 'function.functionId.failure', + 'function.functionId.executions.failure', + 'function.functionId.builds.failure', ]; - $this->foreachDocument($projectId, 'functions', [], function(Document $function) use ($functionMetrics, $projectId) { + $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) { $functionId = $function->getId(); foreach ($functionMetrics as $metric) { $metric = str_replace('functionId', $functionId, $metric); @@ -430,7 +439,7 @@ class UsageDB extends Usage 'users.sessions.delete' ]; - foreach($metrics as $metric) { + foreach ($metrics as $metric) { $this->aggregateDailyMetric($projectId, $metric); } } @@ -439,6 +448,12 @@ class UsageDB extends Usage { $this->aggregateDailyMetric($projectId, 'requests'); $this->aggregateDailyMetric($projectId, 'network'); + $this->aggregateDailyMetric($projectId, 'inbound'); + $this->aggregateDailyMetric($projectId, 'outbound'); + + //Required for billing + $this->aggregateMonthlyMetric($projectId, 'inbound'); + $this->aggregateMonthlyMetric($projectId, 'outbound'); } protected function aggregateDailyMetric(string $projectId, string $metric): void