From e5db5cb4bb8dbc9111f4dbb615246d87bb94606e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 21 Oct 2022 09:39:59 +0000 Subject: [PATCH 01/10] fix console namespace --- src/Appwrite/Usage/Calculators/Database.php | 2 +- src/Appwrite/Usage/Calculators/TimeSeries.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php index 74179fab0b..63ad8b5bf5 100644 --- a/src/Appwrite/Usage/Calculators/Database.php +++ b/src/Appwrite/Usage/Calculators/Database.php @@ -132,7 +132,7 @@ class Database extends Calculator $results = []; $sum = $limit; $latestDocument = null; - $this->database->setNamespace('_' . $projectId); + $this->database->setNamespace($projectId === 'console' ? $projectId : '_' . $projectId); while ($sum === $limit) { try { diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 01c8661206..bd9a36c088 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -301,7 +301,7 @@ class TimeSeries extends Calculator private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_console'); + $this->database->setNamespace('console'); $project = $this->database->getDocument('projects', $projectId); $this->database->setNamespace('_' . $project->getInternalId()); From a385b01d4a1942ee896328b1007e47ddf1f8e70d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 22 Oct 2022 02:25:22 +0000 Subject: [PATCH 02/10] using project db properly --- app/tasks/usage.php | 16 +-- src/Appwrite/Usage/Calculators/Aggregator.php | 104 ++++++++--------- src/Appwrite/Usage/Calculators/Database.php | 108 +++++++++--------- src/Appwrite/Usage/Calculators/TimeSeries.php | 10 +- 4 files changed, 119 insertions(+), 119 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index d1aeab2e84..7068a6a86c 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -9,6 +9,7 @@ use InfluxDB\Database as InfluxDatabase; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; use Utopia\Validator\WhiteList; @@ -50,10 +51,10 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( Console::warning($error->getTraceAsString()); }; -function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void +function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $getProjectDB, callable $logError): void { $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) - $usage = new TimeSeries($database, $influxDB, $logError); + $usage = new TimeSeries($database, $influxDB, $getProjectDB, $logError); Console::loop(function () use ($interval, $usage) { $now = date('d-m-Y H:i:s', time()); @@ -68,11 +69,11 @@ function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, }, $interval); } -function aggregateDatabase(UtopiaDatabase $database, callable $logError): void +function aggregateDatabase(UtopiaDatabase $database, callable $getProjectDB, callable $logError): void { $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) - $usage = new Database($database, $logError); - $aggregrator = new Aggregator($database, $logError); + $usage = new Database($database, $getProjectDB, $logError); + $aggregrator = new Aggregator($database, $getProjectDB, $logError); Console::loop(function () use ($interval, $usage, $aggregrator) { $now = date('d-m-Y H:i:s', time()); @@ -97,13 +98,14 @@ $cli $database = getConsoleDB(); $influxDB = getInfluxDB(); + $getProjectDB = fn (Document $project) => getProjectDB($project); switch ($type) { case 'timeseries': - aggregateTimeseries($database, $influxDB, $logError); + aggregateTimeseries($database, $influxDB, $getProjectDB, $logError); break; case 'database': - aggregateDatabase($database, $logError); + aggregateDatabase($database, $getProjectDB, $logError); break; default: Console::error("Unsupported usage aggregation type"); diff --git a/src/Appwrite/Usage/Calculators/Aggregator.php b/src/Appwrite/Usage/Calculators/Aggregator.php index 67cb18fe56..2121897419 100644 --- a/src/Appwrite/Usage/Calculators/Aggregator.php +++ b/src/Appwrite/Usage/Calculators/Aggregator.php @@ -9,10 +9,8 @@ use Utopia\Database\Query; class Aggregator extends Database { - protected function aggregateDatabaseMetrics(string $projectId): void + protected function aggregateDatabaseMetrics(Document $project): void { - $this->database->setNamespace('_' . $projectId); - $databasesGeneralMetrics = [ 'databases.$all.requests.create', 'databases.$all.requests.read', @@ -29,8 +27,8 @@ class Aggregator extends Database ]; foreach ($databasesGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } $databasesDatabaseMetrics = [ @@ -44,12 +42,12 @@ class Aggregator extends Database 'documents.databaseId.requests.delete', ]; - $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) { + $this->foreachDocument($project, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $project) { $databaseId = $database->getId(); foreach ($databasesDatabaseMetrics as $metric) { $metric = str_replace('databaseId', $databaseId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } $databasesCollectionMetrics = [ @@ -59,21 +57,19 @@ class Aggregator extends Database 'documents.' . $databaseId . '/collectionId.requests.delete', ]; - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) { + $this->foreachDocument($project, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $project) { $collectionId = $collection->getId(); foreach ($databasesCollectionMetrics as $metric) { $metric = str_replace('collectionId', $collectionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } }); }); } - protected function aggregateStorageMetrics(string $projectId): void + protected function aggregateStorageMetrics(Document $project): void { - $this->database->setNamespace('_' . $projectId); - $storageGeneralMetrics = [ 'buckets.$all.requests.create', 'buckets.$all.requests.read', @@ -86,8 +82,8 @@ class Aggregator extends Database ]; foreach ($storageGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } $storageBucketMetrics = [ @@ -97,20 +93,18 @@ class Aggregator extends Database 'files.bucketId.requests.delete', ]; - $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) { + $this->foreachDocument($project, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $project) { $bucketId = $bucket->getId(); foreach ($storageBucketMetrics as $metric) { $metric = str_replace('bucketId', $bucketId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } }); } - protected function aggregateFunctionMetrics(string $projectId): void + protected function aggregateFunctionMetrics(Document $project): void { - $this->database->setNamespace('_' . $projectId); - $functionsGeneralMetrics = [ 'project.$all.compute.total', 'project.$all.compute.time', @@ -125,8 +119,8 @@ class Aggregator extends Database ]; foreach ($functionsGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } $functionMetrics = [ @@ -140,17 +134,17 @@ class Aggregator extends Database 'builds.functionId.compute.time', ]; - $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) { + $this->foreachDocument($project, 'functions', [], function (Document $function) use ($functionMetrics, $project) { $functionId = $function->getId(); foreach ($functionMetrics as $metric) { $metric = str_replace('functionId', $functionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } }); } - protected function aggregateUsersMetrics(string $projectId): void + protected function aggregateUsersMetrics(Document $project): void { $metrics = [ 'users.$all.requests.create', @@ -162,50 +156,50 @@ class Aggregator extends Database ]; foreach ($metrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); + $this->aggregateDailyMetric($project, $metric); + $this->aggregateMonthlyMetric($project, $metric); } } - protected function aggregateGeneralMetrics(string $projectId): void + protected function aggregateGeneralMetrics(Document $project): void { - $this->aggregateDailyMetric($projectId, 'project.$all.network.requests'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.bandwidth'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.inbound'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.outbound'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.requests'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.bandwidth'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.inbound'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.outbound'); + $this->aggregateDailyMetric($project, 'project.$all.network.requests'); + $this->aggregateDailyMetric($project, 'project.$all.network.bandwidth'); + $this->aggregateDailyMetric($project, 'project.$all.network.inbound'); + $this->aggregateDailyMetric($project, 'project.$all.network.outbound'); + $this->aggregateMonthlyMetric($project, 'project.$all.network.requests'); + $this->aggregateMonthlyMetric($project, 'project.$all.network.bandwidth'); + $this->aggregateMonthlyMetric($project, 'project.$all.network.inbound'); + $this->aggregateMonthlyMetric($project, 'project.$all.network.outbound'); } - protected function aggregateDailyMetric(string $projectId, string $metric): void + protected function aggregateDailyMetric(Document $project, string $metric): void { $beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339); $endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339); - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ + $database = call_user_func($this->getProjectDB, $project); + $value = (int) $database->sum('stats', 'value', [ Query::equal('metric', [$metric]), Query::equal('period', ['30m']), Query::greaterThanEqual('time', $beginOfDay), Query::lessThanEqual('time', $endOfDay), ]); - $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); + $this->createOrUpdateMetric($database, $project->getId(), $metric, '1d', $beginOfDay, $value); } - protected function aggregateMonthlyMetric(string $projectId, string $metric): void + protected function aggregateMonthlyMetric(Document $project, string $metric): void { $beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); $endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339); - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ + $database = call_user_func($this->getProjectDB, $project); + $value = (int) $database->sum('stats', 'value', [ Query::equal('metric', [$metric]), Query::equal('period', ['1d']), Query::greaterThanEqual('time', $beginOfMonth), Query::lessThanEqual('time', $endOfMonth), ]); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); + $this->createOrUpdateMetric($database, $project->getId(), $metric, '1mo', $beginOfMonth, $value); } /** @@ -216,16 +210,12 @@ class Aggregator extends Database */ public function collect(): void { - $this->foreachDocument('console', 'projects', [], function (Document $project) { - $projectId = $project->getInternalId(); - - // Aggregate new metrics from already collected usage metrics - // for lower time period (1day and 1 month metric from 30 minute metrics) - $this->aggregateGeneralMetrics($projectId); - $this->aggregateFunctionMetrics($projectId); - $this->aggregateDatabaseMetrics($projectId); - $this->aggregateStorageMetrics($projectId); - $this->aggregateUsersMetrics($projectId); + $this->foreachDocument(new Document(['$id' => 'console']), 'projects', [], function (Document $project) { + $this->aggregateGeneralMetrics($project); + $this->aggregateFunctionMetrics($project); + $this->aggregateDatabaseMetrics($project); + $this->aggregateStorageMetrics($project); + $this->aggregateUsersMetrics($project); }); } } diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php index 63ad8b5bf5..ac8c2876a8 100644 --- a/src/Appwrite/Usage/Calculators/Database.php +++ b/src/Appwrite/Usage/Calculators/Database.php @@ -24,9 +24,10 @@ class Database extends Calculator ], ]; - public function __construct(UtopiaDatabase $database, callable $errorHandler = null) + public function __construct(UtopiaDatabase $database, callable $getProjectDB, callable $errorHandler = null) { $this->database = $database; + $this->getProjectDB = $getProjectDB; $this->errorHandler = $errorHandler; } @@ -35,7 +36,8 @@ class Database extends Calculator * * Create given metric for each defined period * - * @param string $projectId + * @param UtopiaDatabase $database + * @param Document $project * @param string $metric * @param int $value * @param bool $monthly @@ -43,7 +45,7 @@ class Database extends Calculator * @throws Authorization * @throws Structure */ - protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void + protected function createPerPeriodMetric(UtopiaDatabase $database, string $projectId, string $metric, int $value, bool $monthly = false): void { foreach ($this->periods as $options) { $period = $options['key']; @@ -56,13 +58,13 @@ class Database extends Calculator } else { throw new Exception("Period type not found", 500); } - $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); + $this->createOrUpdateMetric($database, $projectId, $metric, $period, $time, $value); } // Required for billing if ($monthly) { $time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value); + $this->createOrUpdateMetric($database, $projectId, $metric, '1mo', $time, $value); } } @@ -71,7 +73,8 @@ class Database extends Calculator * * Create or update each metric in the stats collection for the given project * - * @param string $projectId + * @param UtopiaDatabase $database + * @param String $projectId * @param string $metric * @param string $period * @param string $time @@ -81,15 +84,14 @@ class Database extends Calculator * @throws Authorization * @throws Structure */ - protected function createOrUpdateMetric(string $projectId, string $metric, string $period, string $time, int $value): void + protected function createOrUpdateMetric(UtopiaDatabase $database, String $projectId, string $metric, string $period, string $time, int $value): void { $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_' . $projectId); try { - $document = $this->database->getDocument('stats', $id); + $document = $database->getDocument('stats', $id); if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ + $database->createDocument('stats', new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -98,7 +100,7 @@ class Database extends Calculator 'type' => 2, // these are cumulative metrics ])); } else { - $this->database->updateDocument( + $database->updateDocument( 'stats', $document->getId(), $document->setAttribute('value', $value) @@ -118,7 +120,7 @@ class Database extends Calculator * * Call provided callback for each document in the collection * - * @param string $projectId + * @param Document $project * @param string $collection * @param array $queries * @param callable $callback @@ -126,13 +128,13 @@ class Database extends Calculator * @return void * @throws Exception */ - protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void + protected function foreachDocument(Document $project, string $collection, array $queries, callable $callback): void { $limit = 50; $results = []; $sum = $limit; $latestDocument = null; - $this->database->setNamespace($projectId === 'console' ? $projectId : '_' . $projectId); + $database = $project->getId() == 'console' ? $this->database : call_user_func($this->getProjectDB, $project); while ($sum === $limit) { try { @@ -143,7 +145,7 @@ class Database extends Calculator $results = $this->database->find($collection, \array_merge($paginationQueries, $queries)); } catch (\Exception $e) { if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}"); + call_user_func($this->errorHandler, $e, "fetch_documents_project_{$project->getId()}_collection_{$collection}"); return; } else { throw $e; @@ -169,6 +171,7 @@ class Database extends Calculator * * Calculate sum of an attribute of documents in collection * + * @param UtopiaDatabase $database * @param string $projectId * @param string $collection * @param string $attribute @@ -177,16 +180,15 @@ class Database extends Calculator * @return int * @throws Exception */ - private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int + private function sum(UtopiaDatabase $database, string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int { - $this->database->setNamespace('_' . $projectId); try { - $sum = $this->database->sum($collection, $attribute); + $sum = $database->sum($collection, $attribute); $sum = (int) ($sum * $multiplier); if (!is_null($metric)) { - $this->createPerPeriodMetric($projectId, $metric, $sum); + $this->createPerPeriodMetric($database, $projectId, $metric, $sum); } return $sum; } catch (Exception $e) { @@ -204,6 +206,7 @@ class Database extends Calculator * * Count number of documents in collection * + * @param UtopiaDatabase $database * @param string $projectId * @param string $collection * @param ?string $metric @@ -211,14 +214,14 @@ class Database extends Calculator * @return int * @throws Exception */ - private function count(string $projectId, string $collection, ?string $metric = null): int + private function count(UtopiaDatabase $database, string $projectId, string $collection, ?string $metric = null): int { $this->database->setNamespace('_' . $projectId); try { $count = $this->database->count($collection); if (!is_null($metric)) { - $this->createPerPeriodMetric($projectId, (string) $metric, $count); + $this->createPerPeriodMetric($database, $projectId, (string) $metric, $count); } return $count; } catch (Exception $e) { @@ -236,14 +239,15 @@ class Database extends Calculator * * Total sum of storage used by deployments * + * @param UtopiaDatabase $database * @param string $projectId * * @return int * @throws Exception */ - private function deploymentsTotal(string $projectId): int + private function deploymentsTotal(UtopiaDatabase $database, string $projectId): int { - return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size'); + return $this->sum($database, $projectId, 'deployments', 'size', 'deployments.$all.storage.size'); } /** @@ -251,14 +255,15 @@ class Database extends Calculator * * Metric: users.count * + * @param UtopiaDatabase $database * @param string $projectId * * @return void * @throws Exception */ - private function usersStats(string $projectId): void + private function usersStats(UtopiaDatabase $database, string $projectId): void { - $this->count($projectId, 'users', 'users.$all.count.total'); + $this->count($database, $projectId, 'users', 'users.$all.count.total'); } /** @@ -267,35 +272,36 @@ class Database extends Calculator * Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total, * files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size * - * @param string $projectId + * @param UtopiaDatabase $database + * @param Document $project * * @return void * @throws Authorization * @throws Structure */ - private function storageStats(string $projectId): void + private function storageStats(UtopiaDatabase $database, Document $project): void { $projectFilesTotal = 0; $projectFilesCount = 0; $metric = 'buckets.$all.count.total'; - $this->count($projectId, 'buckets', $metric); + $this->count($database, $project->getId(), 'buckets', $metric); - $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) { + $this->foreachDocument($project, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $project, $database) { $metric = "files.{$bucket->getId()}.count.total"; - $count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric); + $count = $this->count($database, $project->getId(), 'bucket_' . $bucket->getInternalId(), $metric); $projectFilesCount += $count; $metric = "files.{$bucket->getId()}.storage.size"; - $sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric); + $sum = $this->sum($database, $project->getId(), 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric); $projectFilesTotal += $sum; }); - $this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount); - $this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal); + $this->createPerPeriodMetric($database, $project->getId(), 'files.$all.count.total', $projectFilesCount); + $this->createPerPeriodMetric($database, $project->getId(), 'files.$all.storage.size', $projectFilesTotal); - $deploymentsTotal = $this->deploymentsTotal($projectId); - $this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal); + $deploymentsTotal = $this->deploymentsTotal($database, $project->getId()); + $this->createPerPeriodMetric($database, $project->getId(), 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal); } /** @@ -305,38 +311,39 @@ class Database extends Calculator * Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total, * documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total * - * @param string $projectId + * @param UtopiaDatabase $database + * @param Document $project * * @return void * @throws Authorization * @throws Structure */ - private function databaseStats(string $projectId): void + private function databaseStats(UtopiaDatabase $database, Document $project): void { $projectDocumentsCount = 0; $projectCollectionsCount = 0; - $this->count($projectId, 'databases', 'databases.$all.count.total'); + $this->count($database, $project->getId(), 'databases', 'databases.$all.count.total'); - $this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) { + $this->foreachDocument($project, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $project) { $metric = "collections.{$database->getId()}.count.total"; - $count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric); + $count = $this->count($database, $project->getId(), 'database_' . $database->getInternalId(), $metric); $projectCollectionsCount += $count; $databaseDocumentsCount = 0; - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) { + $this->foreachDocument($project, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $project, $database) { $metric = "documents.{$database->getId()}/{$collection->getId()}.count.total"; - $count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric); + $count = $this->count($database, $project->getId(), 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric); $projectDocumentsCount += $count; $databaseDocumentsCount += $count; }); - $this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount); + $this->createPerPeriodMetric($database, $project->getId(), "documents.{$database->getId()}.count.total", $databaseDocumentsCount); }); - $this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount); - $this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount); + $this->createPerPeriodMetric($database, $project->getId(), 'collections.$all.count.total', $projectCollectionsCount); + $this->createPerPeriodMetric($database, $project->getId(), 'documents.$all.count.total', $projectDocumentsCount); } /** @@ -349,12 +356,11 @@ class Database extends Calculator */ public function collect(): void { - $this->foreachDocument('console', 'projects', [], function (Document $project) { - $projectId = $project->getInternalId(); - - $this->usersStats($projectId); - $this->databaseStats($projectId); - $this->storageStats($projectId); + $this->foreachDocument(new Document(['$id' => 'console']), 'projects', [], function (Document $project) { + $database = call_user_func($this->getProjectDB, $project); + $this->usersStats($database, $project->getId()); + $this->databaseStats($database, $project); + $this->storageStats($database, $project); }); } } diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index bd9a36c088..392984c161 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -14,6 +14,7 @@ class TimeSeries extends Calculator protected Database $database; protected $errorHandler; private array $latestTime = []; + private mixed $getProjectDB; // all the mertics that we are collecting protected array $metrics = [ @@ -278,10 +279,11 @@ class TimeSeries extends Calculator 'startTime' => '-24 hours', ]; - public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) + public function __construct(Database $database, InfluxDatabase $influxDB, callable $getProjectDB, callable $errorHandler = null) { $this->database = $database; $this->influxDB = $influxDB; + $this->getProjectDB = $getProjectDB; $this->errorHandler = $errorHandler; } @@ -303,10 +305,10 @@ class TimeSeries extends Calculator $id = \md5("{$time}_{$period}_{$metric}"); $this->database->setNamespace('console'); $project = $this->database->getDocument('projects', $projectId); - $this->database->setNamespace('_' . $project->getInternalId()); + $database = call_user_func($this->getProjectDB, $project); try { - $document = $this->database->getDocument('stats', $id); + $document = $database->getDocument('stats', $id); if ($document->isEmpty()) { $this->database->createDocument('stats', new Document([ '$id' => $id, @@ -317,7 +319,7 @@ class TimeSeries extends Calculator 'type' => $type, ])); } else { - $this->database->updateDocument( + $database->updateDocument( 'stats', $document->getId(), $document->setAttribute('value', $value) From 5b1f8bb2a4feffd6c1c49794eb7739161f097056 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 22 Oct 2022 03:05:19 +0000 Subject: [PATCH 03/10] fix formatting --- src/Appwrite/Usage/Calculators/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php index ac8c2876a8..66ad5a50ed 100644 --- a/src/Appwrite/Usage/Calculators/Database.php +++ b/src/Appwrite/Usage/Calculators/Database.php @@ -84,7 +84,7 @@ class Database extends Calculator * @throws Authorization * @throws Structure */ - protected function createOrUpdateMetric(UtopiaDatabase $database, String $projectId, string $metric, string $period, string $time, int $value): void + protected function createOrUpdateMetric(UtopiaDatabase $database, string $projectId, string $metric, string $period, string $time, int $value): void { $id = \md5("{$time}_{$period}_{$metric}"); From ff6c6003fc2683344db2102bdf6fc903c9395ea4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 22 Oct 2022 03:08:28 +0000 Subject: [PATCH 04/10] get project db for CLI --- app/cli.php | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/app/cli.php b/app/cli.php index 3a62c80816..23cea75306 100644 --- a/app/cli.php +++ b/app/cli.php @@ -12,6 +12,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; use InfluxDB\Database as InfluxDatabase; +use Utopia\Database\Document; function getInfluxDB(): InfluxDatabase { @@ -59,6 +60,29 @@ function getConsoleDB(): Database return $database; } + +function getProjectDB(Document $project): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + function getCache(): Cache { global $register; From e75dfa882bb24dd556a655c6a9aa9672907ce654 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 22 Oct 2022 03:11:35 +0000 Subject: [PATCH 05/10] fix issues --- src/Appwrite/Usage/Calculators/Database.php | 4 +--- src/Appwrite/Usage/Calculators/TimeSeries.php | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php index 66ad5a50ed..8b55e45ecf 100644 --- a/src/Appwrite/Usage/Calculators/Database.php +++ b/src/Appwrite/Usage/Calculators/Database.php @@ -216,10 +216,8 @@ class Database extends Calculator */ private function count(UtopiaDatabase $database, string $projectId, string $collection, ?string $metric = null): int { - $this->database->setNamespace('_' . $projectId); - try { - $count = $this->database->count($collection); + $count = $database->count($collection); if (!is_null($metric)) { $this->createPerPeriodMetric($database, $projectId, (string) $metric, $count); } diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 392984c161..af2da31c6a 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -303,7 +303,6 @@ class TimeSeries extends Calculator private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('console'); $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); From 084b4e8c08fbc20d79c47d44b121be9b1421ea57 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 23 Oct 2022 02:20:26 +0000 Subject: [PATCH 06/10] fix aggregrator --- src/Appwrite/Usage/Calculators/Aggregator.php | 83 ++++++++++--------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/Aggregator.php b/src/Appwrite/Usage/Calculators/Aggregator.php index 2121897419..f0f35526ce 100644 --- a/src/Appwrite/Usage/Calculators/Aggregator.php +++ b/src/Appwrite/Usage/Calculators/Aggregator.php @@ -9,7 +9,7 @@ use Utopia\Database\Query; class Aggregator extends Database { - protected function aggregateDatabaseMetrics(Document $project): void + protected function aggregateDatabaseMetrics(UtopiaDatabase $database, Document $project): void { $databasesGeneralMetrics = [ 'databases.$all.requests.create', @@ -27,8 +27,8 @@ class Aggregator extends Database ]; foreach ($databasesGeneralMetrics as $metric) { - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } $databasesDatabaseMetrics = [ @@ -42,12 +42,12 @@ class Aggregator extends Database 'documents.databaseId.requests.delete', ]; - $this->foreachDocument($project, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $project) { - $databaseId = $database->getId(); + $this->foreachDocument($project, 'databases', [], function (Document $db) use ($databasesDatabaseMetrics, $project, $database) { + $databaseId = $db->getId(); foreach ($databasesDatabaseMetrics as $metric) { $metric = str_replace('databaseId', $databaseId, $metric); - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } $databasesCollectionMetrics = [ @@ -57,18 +57,18 @@ class Aggregator extends Database 'documents.' . $databaseId . '/collectionId.requests.delete', ]; - $this->foreachDocument($project, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $project) { + $this->foreachDocument($project, 'database_' . $db->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $project, $database) { $collectionId = $collection->getId(); foreach ($databasesCollectionMetrics as $metric) { $metric = str_replace('collectionId', $collectionId, $metric); - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } }); }); } - protected function aggregateStorageMetrics(Document $project): void + protected function aggregateStorageMetrics(UtopiaDatabase $database, Document $project): void { $storageGeneralMetrics = [ 'buckets.$all.requests.create', @@ -82,8 +82,8 @@ class Aggregator extends Database ]; foreach ($storageGeneralMetrics as $metric) { - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } $storageBucketMetrics = [ @@ -93,17 +93,17 @@ class Aggregator extends Database 'files.bucketId.requests.delete', ]; - $this->foreachDocument($project, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $project) { + $this->foreachDocument($project, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $project, $database) { $bucketId = $bucket->getId(); foreach ($storageBucketMetrics as $metric) { $metric = str_replace('bucketId', $bucketId, $metric); - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } }); } - protected function aggregateFunctionMetrics(Document $project): void + protected function aggregateFunctionMetrics(UtopiaDatabase $database, Document $project): void { $functionsGeneralMetrics = [ 'project.$all.compute.total', @@ -119,8 +119,8 @@ class Aggregator extends Database ]; foreach ($functionsGeneralMetrics as $metric) { - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } $functionMetrics = [ @@ -134,17 +134,17 @@ class Aggregator extends Database 'builds.functionId.compute.time', ]; - $this->foreachDocument($project, 'functions', [], function (Document $function) use ($functionMetrics, $project) { + $this->foreachDocument($project, 'functions', [], function (Document $function) use ($functionMetrics, $project, $database) { $functionId = $function->getId(); foreach ($functionMetrics as $metric) { $metric = str_replace('functionId', $functionId, $metric); - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } }); } - protected function aggregateUsersMetrics(Document $project): void + protected function aggregateUsersMetrics(UtopiaDatabase $database, Document $project): void { $metrics = [ 'users.$all.requests.create', @@ -156,24 +156,24 @@ class Aggregator extends Database ]; foreach ($metrics as $metric) { - $this->aggregateDailyMetric($project, $metric); - $this->aggregateMonthlyMetric($project, $metric); + $this->aggregateDailyMetric($database, $project, $metric); + $this->aggregateMonthlyMetric($database, $project, $metric); } } - protected function aggregateGeneralMetrics(Document $project): void + protected function aggregateGeneralMetrics(UtopiaDatabase $database, Document $project): void { - $this->aggregateDailyMetric($project, 'project.$all.network.requests'); - $this->aggregateDailyMetric($project, 'project.$all.network.bandwidth'); - $this->aggregateDailyMetric($project, 'project.$all.network.inbound'); - $this->aggregateDailyMetric($project, 'project.$all.network.outbound'); - $this->aggregateMonthlyMetric($project, 'project.$all.network.requests'); - $this->aggregateMonthlyMetric($project, 'project.$all.network.bandwidth'); - $this->aggregateMonthlyMetric($project, 'project.$all.network.inbound'); - $this->aggregateMonthlyMetric($project, 'project.$all.network.outbound'); + $this->aggregateDailyMetric($database, $project, 'project.$all.network.requests'); + $this->aggregateDailyMetric($database, $project, 'project.$all.network.bandwidth'); + $this->aggregateDailyMetric($database, $project, 'project.$all.network.inbound'); + $this->aggregateDailyMetric($database, $project, 'project.$all.network.outbound'); + $this->aggregateMonthlyMetric($database, $project, 'project.$all.network.requests'); + $this->aggregateMonthlyMetric($database, $project, 'project.$all.network.bandwidth'); + $this->aggregateMonthlyMetric($database, $project, 'project.$all.network.inbound'); + $this->aggregateMonthlyMetric($database, $project, 'project.$all.network.outbound'); } - protected function aggregateDailyMetric(Document $project, string $metric): void + protected function aggregateDailyMetric(UtopiaDatabase $database, Document $project, string $metric): void { $beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339); $endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339); @@ -188,7 +188,7 @@ class Aggregator extends Database $this->createOrUpdateMetric($database, $project->getId(), $metric, '1d', $beginOfDay, $value); } - protected function aggregateMonthlyMetric(Document $project, string $metric): void + protected function aggregateMonthlyMetric(UtopiaDatabase $database, Document $project, string $metric): void { $beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); $endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339); @@ -211,11 +211,12 @@ class Aggregator extends Database public function collect(): void { $this->foreachDocument(new Document(['$id' => 'console']), 'projects', [], function (Document $project) { - $this->aggregateGeneralMetrics($project); - $this->aggregateFunctionMetrics($project); - $this->aggregateDatabaseMetrics($project); - $this->aggregateStorageMetrics($project); - $this->aggregateUsersMetrics($project); + $database = call_user_func($this->getProjectDB, $project); + $this->aggregateGeneralMetrics($database, $project); + $this->aggregateFunctionMetrics($database, $project); + $this->aggregateDatabaseMetrics($database, $project); + $this->aggregateStorageMetrics($database, $project); + $this->aggregateUsersMetrics($database, $project); }); } } From adf3f74ef2f1baacf92839d9667d3021b7206a98 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 23 Oct 2022 08:19:10 +0000 Subject: [PATCH 07/10] refill pools on exception --- app/cli.php | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/app/cli.php b/app/cli.php index 23cea75306..94830490af 100644 --- a/app/cli.php +++ b/app/cli.php @@ -47,11 +47,20 @@ function getConsoleDB(): Database $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource() - ; + try { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + } catch (Throwable $error) { + $pools->fill(); + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + } $database = new Database($dbAdapter, getCache()); @@ -71,11 +80,20 @@ function getProjectDB(Document $project): Database return getConsoleDB(); } - $dbAdapter = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; + try { + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + } catch (Throwable $error) { + $pools->fill(); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + } $database = new Database($dbAdapter, getCache()); $database->setNamespace('_' . $project->getInternalId()); From 70afd1efeddccedbf6c3abfc1c8578aa1b8ea8bc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 24 Oct 2022 01:34:12 +0000 Subject: [PATCH 08/10] pool reclaim --- app/cli.php | 38 ++++++++++---------------------------- app/tasks/usage.php | 3 ++- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/app/cli.php b/app/cli.php index 94830490af..23cea75306 100644 --- a/app/cli.php +++ b/app/cli.php @@ -47,20 +47,11 @@ function getConsoleDB(): Database $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ - try { - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource() - ; - } catch (Throwable $error) { - $pools->fill(); - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource() - ; - } + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; $database = new Database($dbAdapter, getCache()); @@ -80,20 +71,11 @@ function getProjectDB(Document $project): Database return getConsoleDB(); } - try { - $dbAdapter = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; - } catch (Throwable $error) { - $pools->fill(); - $dbAdapter = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; - } + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; $database = new Database($dbAdapter, getCache()); $database->setNamespace('_' . $project->getInternalId()); diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 7068a6a86c..049041bdc8 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -92,7 +92,7 @@ $cli ->task('usage') ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function (string $type) use ($logError) { + ->action(function (string $type) use ($logError, $register) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); @@ -110,4 +110,5 @@ $cli default: Console::error("Unsupported usage aggregation type"); } + $register->get('pools')->reclaim(); }); From be003bb5e67f4ebb2304cb78b973542b31fe6f20 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 24 Oct 2022 02:23:26 +0000 Subject: [PATCH 09/10] fix openapi spec test --- tests/e2e/General/HTTPTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 06ceada976..1e26396a0e 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -186,7 +186,8 @@ class HTTPTest extends Scope $response['body'] = json_decode($response['body'], true); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertTrue(empty($response['body'])); + // looks like recent change in the validator + $this->assertTrue(empty($response['body']['schemaValidationMessages'])); } } From 4dc6e2fb6aeb3e922772281ea5c7a6322d69a0d0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 24 Oct 2022 02:33:19 +0000 Subject: [PATCH 10/10] use register to reclaim --- app/tasks/usage.php | 9 +++++---- src/Appwrite/Usage/Calculators/Aggregator.php | 1 + src/Appwrite/Usage/Calculators/Database.php | 6 +++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 049041bdc8..c47850bc1e 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -12,6 +12,7 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; +use Utopia\Registry\Registry; use Utopia\Validator\WhiteList; Authorization::disable(); @@ -69,11 +70,11 @@ function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, }, $interval); } -function aggregateDatabase(UtopiaDatabase $database, callable $getProjectDB, callable $logError): void +function aggregateDatabase(UtopiaDatabase $database, callable $getProjectDB, Registry $register, callable $logError): void { $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) - $usage = new Database($database, $getProjectDB, $logError); - $aggregrator = new Aggregator($database, $getProjectDB, $logError); + $usage = new Database($database, $getProjectDB, $register, $logError); + $aggregrator = new Aggregator($database, $getProjectDB, $register, $logError); Console::loop(function () use ($interval, $usage, $aggregrator) { $now = date('d-m-Y H:i:s', time()); @@ -105,7 +106,7 @@ $cli aggregateTimeseries($database, $influxDB, $getProjectDB, $logError); break; case 'database': - aggregateDatabase($database, $getProjectDB, $logError); + aggregateDatabase($database, $getProjectDB, $register, $logError); break; default: Console::error("Unsupported usage aggregation type"); diff --git a/src/Appwrite/Usage/Calculators/Aggregator.php b/src/Appwrite/Usage/Calculators/Aggregator.php index f0f35526ce..5450ff6440 100644 --- a/src/Appwrite/Usage/Calculators/Aggregator.php +++ b/src/Appwrite/Usage/Calculators/Aggregator.php @@ -217,6 +217,7 @@ class Aggregator extends Database $this->aggregateDatabaseMetrics($database, $project); $this->aggregateStorageMetrics($database, $project); $this->aggregateUsersMetrics($database, $project); + $this->register->get('pools')->reclaim(); }); } } diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php index 8b55e45ecf..ce1c3755e5 100644 --- a/src/Appwrite/Usage/Calculators/Database.php +++ b/src/Appwrite/Usage/Calculators/Database.php @@ -10,9 +10,11 @@ use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Database\Query; +use Utopia\Registry\Registry; class Database extends Calculator { + protected Registry $register; protected array $periods = [ [ 'key' => '30m', @@ -24,8 +26,9 @@ class Database extends Calculator ], ]; - public function __construct(UtopiaDatabase $database, callable $getProjectDB, callable $errorHandler = null) + public function __construct(UtopiaDatabase $database, callable $getProjectDB, Registry $register, callable $errorHandler = null) { + $this->register = $register; $this->database = $database; $this->getProjectDB = $getProjectDB; $this->errorHandler = $errorHandler; @@ -359,6 +362,7 @@ class Database extends Calculator $this->usersStats($database, $project->getId()); $this->databaseStats($database, $project); $this->storageStats($database, $project); + $this->register->get('pools')->reclaim(); }); } }