diff --git a/composer.lock b/composer.lock index 93b8455cac..bce5b90d88 100644 --- a/composer.lock +++ b/composer.lock @@ -5370,5 +5370,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php index 083ff721fc..560bd0ff0f 100644 --- a/src/Appwrite/Stats/Usage.php +++ b/src/Appwrite/Stats/Usage.php @@ -28,9 +28,6 @@ class Usage 'outbound' => [ 'table' => 'appwrite_usage_network_outbound', ], - 'executions' => [ - 'table' => 'appwrite_usage_executions_all', - ], 'databases.create' => [ 'table' => 'appwrite_usage_databases_create', ], @@ -225,17 +222,9 @@ class Usage ], ]; - protected array $periods = [ - [ - 'key' => '30m', - 'multiplier' => 1800, - 'startTime' => '-24 hours', - ], - [ - 'key' => '1d', - 'multiplier' => 86400, - 'startTime' => '-90 days', - ], + protected array $period = [ + 'key' => '30m', + 'startTime' => '-24 hours', ]; public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) @@ -378,15 +367,13 @@ class Usage public function collect(): void { foreach ($this->metrics as $metric => $options) { //for each metrics - foreach ($this->periods as $period) { // aggregate data for each period - try { - $this->syncFromInfluxDB($metric, $options, $period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; - } + try { + $this->syncFromInfluxDB($metric, $options, $this->period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; } } } diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php index 884ad7c4b3..1caef7acdb 100644 --- a/src/Appwrite/Stats/UsageDB.php +++ b/src/Appwrite/Stats/UsageDB.php @@ -4,14 +4,51 @@ namespace Appwrite\Stats; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; class UsageDB extends Usage { + protected array $periods = [ + [ + 'key' => '30m', + 'multiplier' => 1800, + ], + [ + 'key' => '1d', + 'multiplier' => 86400, + ], + ]; + public function __construct(Database $database, callable $errorHandler = null) { $this->database = $database; $this->errorHandler = $errorHandler; } + /** + * Create Per Period Metric + * Create given metric for each defined period + * + * @param string $projectId + * @param string $metric + * @param int $value + * + * @return void + */ + private function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void + { + foreach ($this->periods as $options) { + $period = $options['key']; + $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); + $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); + } + + // Required for billing + if ($monthly) { + $time = strtotime("first day of the month"); + $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value); + } + } + /** * Create or Update Mertic * Create or update each metric in the stats collection for the given project @@ -22,38 +59,34 @@ class UsageDB extends Usage * * @return void */ - private function createOrUpdateMetric(string $projectId, string $metric, int $value): void + private function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void { - foreach ($this->periods as $options) { - $period = $options['key']; - $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); $id = \md5("{$time}_{$period}_{$metric}"); $this->database->setNamespace('_' . $projectId); - try { - $document = $this->database->getDocument('stats', $id); - if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 1, - ])); - } else { - $this->database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - } catch (\Exception$e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } + try { + $document = $this->database->getDocument('stats', $id); + if ($document->isEmpty()) { + $this->database->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 1, + ])); + } else { + $this->database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception$e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; } } } @@ -111,7 +144,6 @@ class UsageDB extends Usage * @param string $collection * @param string $attribute * @param string $metric - * @param int $multiplier * * @return int */ @@ -122,7 +154,7 @@ class UsageDB extends Usage try { $sum = $this->database->sum($collection, $attribute); $sum = (int) ($sum * $multiplier); - $this->createOrUpdateMetric($projectId, $metric, $sum); + $this->createPerPeriodMetric($projectId, $metric, $sum); return $sum; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { @@ -150,7 +182,7 @@ class UsageDB extends Usage try { $count = $this->database->count($collection); - $this->createOrUpdateMetric($projectId, $metric, $count); + $this->createPerPeriodMetric($projectId, $metric, $count); return $count; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { @@ -218,16 +250,15 @@ class UsageDB extends Usage $projectFilesTotal += $sum; }); - $this->createOrUpdateMetric($projectId, 'storage.files.count', $projectFilesCount); - $this->createOrUpdateMetric($projectId, 'storage.files.total', $projectFilesTotal); + $this->createPerPeriodMetric($projectId, 'storage.files.count', $projectFilesCount); + $this->createPerPeriodMetric($projectId, 'storage.files.total', $projectFilesTotal); - $this->createOrUpdateMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); + $this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); } /** - * Storage Stats - * Metrics: storage.total, storage.files.total, storage.buckets.{bucketId}.files.total, - * storage.buckets.count, storage.files.count, storage.buckets.{bucketId}.files.count + * Compute Stats + * Metrics: functions.executionTime, functions.buildTime, functions.compute, * * @param string $projectId * @@ -238,7 +269,7 @@ class UsageDB extends Usage $executionTotal = $this->sum($projectId, 'executions', 'time', 'functions.executionTime', 1000); // in ms $buildTotal = $this->sum($projectId, 'builds', 'duration', 'functions.buildTime', 1000); // in ms - $this->createOrUpdateMetric($projectId, 'functions.compute', $executionTotal + $buildTotal); //in ms + $this->createPerPeriodMetric($projectId, 'functions.compute', $executionTotal + $buildTotal); //in ms } /** @@ -272,11 +303,185 @@ class UsageDB extends Usage $databaseDocumentsCount += $count; }); - $this->createOrUpdateMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount); + $this->createPerPeriodMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount); }); - $this->createOrUpdateMetric($projectId, 'databases.collections.count', $projectCollectionsCount); - $this->createOrUpdateMetric($projectId, 'databases.documents.count', $projectDocumentsCount); + $this->createPerPeriodMetric($projectId, 'databases.collections.count', $projectCollectionsCount); + $this->createPerPeriodMetric($projectId, 'databases.documents.count', $projectDocumentsCount); + } + + protected function aggregateDatabaseMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $databasesGeneralMetrics = [ + 'databases.create', + 'databases.read', + 'databases.update', + 'databases.delete', + 'databases.collections.create', + 'databases.collections.read', + 'databases.collections.update', + 'databases.collections.delete', + 'databases.documents.create', + 'databases.documents.read', + 'databases.documents.update', + 'databases.documents.delete', + ]; + + foreach ($databasesGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesDatabaseMetrics = [ + 'databases.databaseId.collections.create', + 'databases.databaseId.collections.read', + 'databases.databaseId.collections.update', + 'databases.databaseId.collections.delete', + 'databases.databaseId.documents.create', + 'databases.databaseId.documents.read', + 'databases.databaseId.documents.update', + 'databases.databaseId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) { + $databaseId = $database->getId(); + foreach ($databasesDatabaseMetrics as $metric) { + $metric = str_replace('databaseId', $databaseId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesCollectionMetrics = [ + 'databases.' . $databaseId . '.collections.collectionId.documents.create', + 'databases.' . $databaseId . '.collections.collectionId.documents.read', + 'databases.' . $databaseId . '.collections.collectionId.documents.update', + 'databases.' . $databaseId . '.collections.collectionId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) { + $collectionId = $collection->getId(); + foreach ($databasesCollectionMetrics as $metric) { + $metric = str_replace('collectionId', $collectionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + }); + } + + protected function aggregateStorageMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $storageGeneralMetrics = [ + 'storage.buckets.create', + 'storage.buckets.read', + 'storage.buckets.update', + 'storage.buckets.delete', + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach ($storageGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $storageBucketMetrics = [ + 'storage.buckets.bucketId.files.create', + 'storage.buckets.bucketId.files.read', + 'storage.buckets.bucketId.files.update', + 'storage.buckets.bucketId.files.delete', + ]; + + $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) { + $bucketId = $bucket->getId(); + foreach ($storageBucketMetrics as $metric) { + $metric = str_replace('bucketId', $bucketId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateFunctionMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $this->aggregateDailyMetric($projectId, 'functions.executions'); + $this->aggregateDailyMetric($projectId, 'functions.builds'); + $this->aggregateDailyMetric($projectId, 'functions.failures'); + + $functionMetrics = [ + 'functions.functionId.executions', + 'functions.functionId.builds', + 'functions.functionId.compute', + 'function.functionId.executions.failure', + 'function.functionId.builds.failure', + ]; + + $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) { + $functionId = $function->getId(); + foreach ($functionMetrics as $metric) { + $metric = str_replace('functionId', $functionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateUsersMetrics(string $projectId): void + { + $metrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + 'users.sessions.create', + 'users.sessions.delete' + ]; + + foreach ($metrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + } + + protected function aggregateGeneralMetrics(string $projectId): void + { + $this->aggregateDailyMetric($projectId, 'requests'); + $this->aggregateDailyMetric($projectId, 'network'); + $this->aggregateDailyMetric($projectId, 'inbound'); + $this->aggregateDailyMetric($projectId, 'outbound'); + + //Required for billing + $this->aggregateMonthlyMetric($projectId, 'inbound'); + $this->aggregateMonthlyMetric($projectId, 'outbound'); + } + + protected function aggregateDailyMetric(string $projectId, string $metric): void + { + $beginOfDay = strtotime("today"); + $endOfDay = strtotime("tomorrow", $beginOfDay) - 1; + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); + } + + protected function aggregateMonthlyMetric(string $projectId, string $metric): void + { + $beginOfMonth = strtotime("first day of the month"); + $endOfMonth = strtotime("last day of the month"); + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['1d']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); } /** @@ -289,10 +494,19 @@ class UsageDB extends Usage { $this->foreachDocument('console', 'projects', [], function (Document $project) { $projectId = $project->getInternalId(); - $this->computeStats($projectId); + $this->usersStats($projectId); $this->databaseStats($projectId); $this->storageStats($projectId); + $this->computeStats($projectId); + + // Aggregate new metrics from already collected usage metrics + // for lower time period (1day and 1 month metric from 30 minute metrics) + $this->aggregateGeneralMetrics($projectId); + $this->aggregateFunctionMetrics($projectId); + $this->aggregateDatabaseMetrics($projectId); + $this->aggregateStorageMetrics($projectId); + $this->aggregateUsersMetrics($projectId); }); } }