From 6dfe1eb38975bd4449e37be18b5349224b90ddb4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 18 Feb 2022 03:22:13 +0000 Subject: [PATCH 1/6] add one more try catch to handle database not found --- app/tasks/usage.php | 86 ++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 02b1b7de1f..5d19b52ec5 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -316,51 +316,55 @@ $cli } $query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)"; - $result = $database->query($query); - - $points = $result->getPoints(); - foreach ($points as $point) { - $projectId = $point['projectId']; - - if (!empty($projectId) && $projectId !== 'console') { - $dbForProject->setNamespace('_project_' . $projectId); - $metricUpdated = $metric; - - if (!empty($groupBy)) { - $groupedBy = $point[$options['groupBy']] ?? ''; - if (empty($groupedBy)) { - continue; + try { + $result = $database->query($query); + + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + + if (!empty($projectId) && $projectId !== 'console') { + $dbForProject->setNamespace('_project_' . $projectId); + $metricUpdated = $metric; + + if (!empty($groupBy)) { + $groupedBy = $point[$options['groupBy']] ?? ''; + if (empty($groupedBy)) { + continue; + } + $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); } - $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); - } - - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric - $value = (!empty($point['value'])) ? $point['value'] : 0; - - try { - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period['key'], - 'time' => $time, - 'metric' => $metricUpdated, - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); + + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric + $value = (!empty($point['value'])) ? $point['value'] : 0; + + try { + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period['key'], + 'time' => $time, + 'metric' => $metricUpdated, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + $latestTime[$metric][$period['key']] = $time; + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); } - $latestTime[$metric][$period['key']] = $time; - } catch (\Exception $e) { // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); } } + } catch (\Exception $e) { + Console::warning("Failed to Query: {$e->getMessage()}"); } } } From fe5ff5b12ca99710235da9a5e960ffa1c08a14b5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 20 Feb 2022 09:27:23 +0000 Subject: [PATCH 2/6] update cli and orchentration version --- composer.json | 4 +- composer.lock | 203 ++++++++++++++++---------------------------------- 2 files changed, 66 insertions(+), 141 deletions(-) diff --git a/composer.json b/composer.json index 819aa53c07..db6c354b62 100644 --- a/composer.json +++ b/composer.json @@ -44,11 +44,11 @@ "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.8.*", "utopia-php/cache": "0.4.*", - "utopia-php/cli": "0.11.*", + "utopia-php/cli": "0.12.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.14.*", "utopia-php/locale": "0.4.*", - "utopia-php/orchestration": "0.2.*", + "utopia-php/orchestration": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", diff --git a/composer.lock b/composer.lock index 3da06db4c8..4d796a4104 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0a0bb53e6a5daeb3afde139b8199da3b", + "content-hash": "71d2e3bdd2ee9ed2bd4c8ae2e62ea1a2", "packages": [ { "name": "adhocore/jwt", @@ -531,12 +531,12 @@ } }, "autoload": { - "psr-4": { - "GuzzleHttp\\": "src/" - }, "files": [ "src/functions_include.php" - ] + ], + "psr-4": { + "GuzzleHttp\\": "src/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -2037,16 +2037,16 @@ }, { "name": "utopia-php/cli", - "version": "0.11.0", + "version": "0.12.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "c7a6908a8dbe9234b8b2c954e5487d34cb079af6" + "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/c7a6908a8dbe9234b8b2c954e5487d34cb079af6", - "reference": "c7a6908a8dbe9234b8b2c954e5487d34cb079af6", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/6d164b752efeb1ca089e3a517bc274d8b383474b", + "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b", "shasum": "" }, "require": { @@ -2084,9 +2084,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.11.0" + "source": "https://github.com/utopia-php/cli/tree/0.12.0" }, - "time": "2021-04-16T15:16:08+00:00" + "time": "2022-02-18T22:10:41+00:00" }, { "name": "utopia-php/config", @@ -2258,16 +2258,16 @@ }, { "name": "utopia-php/framework", - "version": "0.19.6", + "version": "0.19.7", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "7d9b28365fb794001cb34dd028659452d4e71b7d" + "reference": "f17afe77a21873b9be18ebc05283813468b4283a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/7d9b28365fb794001cb34dd028659452d4e71b7d", - "reference": "7d9b28365fb794001cb34dd028659452d4e71b7d", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/f17afe77a21873b9be18ebc05283813468b4283a", + "reference": "f17afe77a21873b9be18ebc05283813468b4283a", "shasum": "" }, "require": { @@ -2301,9 +2301,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.6" + "source": "https://github.com/utopia-php/framework/tree/0.19.7" }, - "time": "2022-02-10T17:05:22+00:00" + "time": "2022-02-18T00:04:49+00:00" }, { "name": "utopia-php/image", @@ -2476,21 +2476,21 @@ }, { "name": "utopia-php/orchestration", - "version": "0.2.1", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "55da7a331a45d5887de8122268dfccd15fee94d1" + "reference": "67cf0ab15a096d274c093ea918aa4ace14ac7af7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/55da7a331a45d5887de8122268dfccd15fee94d1", - "reference": "55da7a331a45d5887de8122268dfccd15fee94d1", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/67cf0ab15a096d274c093ea918aa4ace14ac7af7", + "reference": "67cf0ab15a096d274c093ea918aa4ace14ac7af7", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/cli": "0.11.*" + "utopia-php/cli": "0.12.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -2525,9 +2525,9 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/0.2.1" + "source": "https://github.com/utopia-php/orchestration/tree/0.4.1" }, - "time": "2021-09-03T11:29:20+00:00" + "time": "2022-02-20T09:23:06+00:00" }, { "name": "utopia-php/preloader", @@ -2636,16 +2636,16 @@ }, { "name": "utopia-php/storage", - "version": "0.6.0", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "ad628025be3b3e0818ea1e6cb701cc08a1c52344" + "reference": "de12487f77346475e2deed82212c6f842c9ae4a8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/ad628025be3b3e0818ea1e6cb701cc08a1c52344", - "reference": "ad628025be3b3e0818ea1e6cb701cc08a1c52344", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/de12487f77346475e2deed82212c6f842c9ae4a8", + "reference": "de12487f77346475e2deed82212c6f842c9ae4a8", "shasum": "" }, "require": { @@ -2682,9 +2682,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.6.0" + "source": "https://github.com/utopia-php/storage/tree/0.7.0" }, - "time": "2022-01-31T06:29:51+00:00" + "time": "2022-02-09T13:02:47+00:00" }, { "name": "utopia-php/swoole", @@ -3087,7 +3087,7 @@ "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator", - "reference": "96735eef25d5a98d0582bb4925b42db593c974f8" + "reference": "b977fcf357a267f41299539ac9095aa7bbcc2600" }, "require": { "ext-curl": "*", @@ -3095,10 +3095,10 @@ "ext-mbstring": "*", "matthiasmullie/minify": "^1.3", "php": ">=7.0.0", - "twig/twig": "^2.14" + "twig/twig": "^3.3" }, "require-dev": { - "phpunit/phpunit": "^7.0" + "phpunit/phpunit": "^9.5.13" }, "type": "library", "autoload": { @@ -3107,6 +3107,11 @@ "Appwrite\\Spec\\": "src/Spec" } }, + "autoload-dev": { + "psr-4": { + "Tests\\": "tests" + } + }, "license": [ "MIT" ], @@ -3117,7 +3122,7 @@ } ], "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", - "time": "2022-01-30T07:29:08+00:00" + "time": "2022-02-15T11:09:40+00:00" }, { "name": "composer/pcre", @@ -4230,16 +4235,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.10", + "version": "9.2.11", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "d5850aaf931743067f4bfc1ae4cbd06468400687" + "reference": "665a1ac0a763c51afc30d6d130dac0813092b17f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d5850aaf931743067f4bfc1ae4cbd06468400687", - "reference": "d5850aaf931743067f4bfc1ae4cbd06468400687", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/665a1ac0a763c51afc30d6d130dac0813092b17f", + "reference": "665a1ac0a763c51afc30d6d130dac0813092b17f", "shasum": "" }, "require": { @@ -4295,7 +4300,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.10" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.11" }, "funding": [ { @@ -4303,7 +4308,7 @@ "type": "github" } ], - "time": "2021-12-05T09:12:13+00:00" + "time": "2022-02-18T12:46:09+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5208,16 +5213,16 @@ }, { "name": "sebastian/global-state", - "version": "5.0.4", + "version": "5.0.5", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "19c519631c5a511b7ed0ad64a6713fdb3fd25fe4" + "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/19c519631c5a511b7ed0ad64a6713fdb3fd25fe4", - "reference": "19c519631c5a511b7ed0ad64a6713fdb3fd25fe4", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2", + "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2", "shasum": "" }, "require": { @@ -5260,7 +5265,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.4" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5" }, "funding": [ { @@ -5268,7 +5273,7 @@ "type": "github" } ], - "time": "2022-02-10T07:01:19+00:00" + "time": "2022-02-14T08:28:10+00:00" }, { "name": "sebastian/lines-of-code", @@ -6002,12 +6007,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Mbstring\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -6051,82 +6056,6 @@ ], "time": "2021-11-30T18:21:41+00:00" }, - { - "name": "symfony/polyfill-php72", - "version": "v1.24.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php72.git", - "reference": "9a142215a36a3888e30d0a9eeea9766764e96976" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php72/zipball/9a142215a36a3888e30d0a9eeea9766764e96976", - "reference": "9a142215a36a3888e30d0a9eeea9766764e96976", - "shasum": "" - }, - "require": { - "php": ">=7.1" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-main": "1.23-dev" - }, - "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Php72\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 7.2+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-php72/tree/v1.24.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2021-05-27T09:17:38+00:00" - }, { "name": "symfony/service-contracts", "version": "v3.0.0", @@ -6395,23 +6324,22 @@ }, { "name": "twig/twig", - "version": "v2.14.11", + "version": "v3.3.8", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "66baa66f29ee30e487e05f1679903e36eb01d727" + "reference": "972d8604a92b7054828b539f2febb0211dd5945c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/66baa66f29ee30e487e05f1679903e36eb01d727", - "reference": "66baa66f29ee30e487e05f1679903e36eb01d727", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/972d8604a92b7054828b539f2febb0211dd5945c", + "reference": "972d8604a92b7054828b539f2febb0211dd5945c", "shasum": "" }, "require": { - "php": ">=7.1.3", + "php": ">=7.2.5", "symfony/polyfill-ctype": "^1.8", - "symfony/polyfill-mbstring": "^1.3", - "symfony/polyfill-php72": "^1.8" + "symfony/polyfill-mbstring": "^1.3" }, "require-dev": { "psr/container": "^1.0", @@ -6420,13 +6348,10 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.14-dev" + "dev-master": "3.3-dev" } }, "autoload": { - "psr-0": { - "Twig_": "lib/" - }, "psr-4": { "Twig\\": "src/" } @@ -6459,7 +6384,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v2.14.11" + "source": "https://github.com/twigphp/Twig/tree/v3.3.8" }, "funding": [ { @@ -6471,7 +6396,7 @@ "type": "tidelift" } ], - "time": "2022-02-04T06:57:25+00:00" + "time": "2022-02-04T06:59:48+00:00" }, { "name": "vimeo/psalm", From 8707938c759e8c09eac090681be44201a3ab2c5b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 20 Feb 2022 09:33:01 +0000 Subject: [PATCH 3/6] some cleanup --- app/tasks/usage.php | 635 ++++++++++++++++++++++---------------------- 1 file changed, 319 insertions(+), 316 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 5d19b52ec5..a81e5bbc20 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -374,116 +374,290 @@ $cli * Aggregate MariaDB every 15 minutes * Some of the queries here might contain full-table scans. */ - if ($iterations % 30 === 0) { // Every 15 minutes aggregate number of objects in database + if ($iterations % 30 != 0) { // Aggregate aggregate number of objects in database only after 15 minutes + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + return; + } + $latestProject = null; - $latestProject = null; + do { // Loop over all the projects + $attempts = 0; + $max = 10; + $sleep = 1; - do { // Loop over all the projects - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // list projects - try { - $attempts++; - $projects = $dbForConsole->find('projects', [], 100, cursor: $latestProject); - break; // leave the do-while if successful - } catch (\Exception $e) { - Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed access console db: ' . $e->getMessage()); - } - sleep($sleep); + do { // list projects + try { + $attempts++; + $projects = $dbForConsole->find('projects', [], 100, cursor: $latestProject); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed access console db: ' . $e->getMessage()); } - } while ($attempts < $max); + sleep($sleep); + } + } while ($attempts < $max); - if (empty($projects)) { - continue; + if (empty($projects)) { + continue; + } + + $latestProject = $projects[array_key_last($projects)]; + + foreach ($projects as $project) { + $projectId = $project->getId(); + + // Get total storage + $dbForProject->setNamespace('_project_' . $projectId); + $storageTotal = $dbForProject->sum('tags', 'size'); + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_storage.tags.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'storage.tags.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $storageTotal) + ); } - $latestProject = $projects[array_key_last($projects)]; + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_storage.tags.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '1d', + 'time' => $time, + 'metric' => 'storage.tags.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $storageTotal) + ); + } - foreach ($projects as $project) { - $projectId = $project->getId(); - - // Get total storage - $dbForProject->setNamespace('_project_' . $projectId); - $storageTotal = $dbForProject->sum('tags', 'size'); - - $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes - $id = \md5($time . '_30m_storage.tags.total'); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '30m', - 'time' => $time, - 'metric' => 'storage.tags.total', - 'value' => $storageTotal, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $storageTotal) - ); - } - - $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day - $id = \md5($time . '_1d_storage.tags.total'); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '1d', - 'time' => $time, - 'metric' => 'storage.tags.total', - 'value' => $storageTotal, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $storageTotal) - ); - } - - $collections = [ - 'users' => [ - 'namespace' => '', - ], - 'collections' => [ - 'metricPrefix' => 'database', - 'namespace' => '', - 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting - 'documents' => [ - 'namespace' => '', - ], + $collections = [ + 'users' => [ + 'namespace' => '', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => '', + 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting + 'documents' => [ + 'namespace' => '', ], ], - 'buckets' => [ - 'metricPrefix' => 'storage', - 'namespace' => '', - 'subCollections' => [ - 'files' => [ - 'namespace' => '', - 'collectionPrefix' => 'bucket_', - 'sum' => [ - 'field' => 'sizeOriginal' - ] - ], - ] + ], + 'buckets' => [ + 'metricPrefix' => 'storage', + 'namespace' => '', + 'subCollections' => [ + 'files' => [ + 'namespace' => '', + 'collectionPrefix' => 'bucket_', + 'sum' => [ + 'field' => 'sizeOriginal' + ] + ], ] - ]; + ] + ]; - foreach ($collections as $collection => $options) { - try { + foreach ($collections as $collection => $options) { + try { + $dbForProject->setNamespace("_project_{$projectId}"); + $count = $dbForProject->count($collection); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $count) + ); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $count) + ); + } + + $subCollections = $options['subCollections'] ?? []; + + if (empty($subCollections)) { + continue; + } + + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + $subCollectionTotals = []; //total project level sum of sub collections + + do { // Loop over all the parent collection document for each sub collection $dbForProject->setNamespace("_project_{$projectId}"); - $count = $dbForProject->count($collection); - $metricPrefix = $options['metricPrefix'] ?? ''; - $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + $parents = $dbForProject->find($collection, [], 100, cursor: $latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections + + if (empty($parents)) { + continue; + } + + $latestParent = $parents[array_key_last($parents)]; + + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count + $dbForProject->setNamespace("_project_{$projectId}"); + $count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getId()); + + $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count + + $dbForProject->setNamespace("_project_{$projectId}"); + + $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $count) + ); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $count) + ); + } + + // check if sum calculation is required + $sum = $subOptions['sum'] ?? []; + if(empty($sum)) { + continue; + } + + $dbForProject->setNamespace("_project_{$projectId}"); + $total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getId(), $sum['field']); + + $subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total + + $dbForProject->setNamespace("_project_{$projectId}"); + + $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.total"; + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $total, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $total)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $total, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $total)); + } + + } + } + } while (!empty($parents)); + + /** + * Inserting project level counts for sub collections like database.documents.count + */ + foreach ($subCollectionCounts as $subCollection => $count) { + $dbForProject->setNamespace("_project_{$projectId}"); + + $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric @@ -524,227 +698,56 @@ $cli $document->setAttribute('value', $count) ); } - - $subCollections = $options['subCollections'] ?? []; - - if (empty($subCollections)) { - continue; - } - - $latestParent = null; - $subCollectionCounts = []; //total project level count of sub collections - $subCollectionTotals = []; //total project level sum of sub collections - - do { // Loop over all the parent collection document for each sub collection - $dbForProject->setNamespace("_project_{$projectId}"); - $parents = $dbForProject->find($collection, [], 100, cursor: $latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections - - if (empty($parents)) { - continue; - } - - $latestParent = $parents[array_key_last($parents)]; - - foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count - $dbForProject->setNamespace("_project_{$projectId}"); - $count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getId()); - - $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count - - $dbForProject->setNamespace("_project_{$projectId}"); - - $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; - $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes - $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '30m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $count) - ); - } - - $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day - $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '1d', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $count) - ); - } - - // check if sum calculation is required - $sum = $subOptions['sum'] ?? []; - if(empty($sum)) { - continue; - } - - $dbForProject->setNamespace("_project_{$projectId}"); - $total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getId(), $sum['field']); - - $subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total - - $dbForProject->setNamespace("_project_{$projectId}"); - - $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.total"; - $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes - $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '30m', - 'metric' => $metric, - 'value' => $total, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $total)); - } - - $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day - $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '1d', - 'metric' => $metric, - 'value' => $total, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $total)); - } - - } - } - } while (!empty($parents)); - - /** - * Inserting project level counts for sub collections like database.documents.count - */ - foreach ($subCollectionCounts as $subCollection => $count) { - $dbForProject->setNamespace("_project_{$projectId}"); - - $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; - - $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes - $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '30m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $count) - ); - } - - $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day - $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '1d', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $count) - ); - } - } - - /** - * Inserting project level sums for sub collections like storage.total - */ - foreach ($subCollectionTotals as $subCollection => $count) { - $dbForProject->setNamespace("_project_{$projectId}"); - - $metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total"; - - $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes - $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '30m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $count)); - } - - $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day - $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'time' => $time, - 'period' => '1d', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $count)); - } - } - } catch (\Exception$e) { - Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } + + /** + * Inserting project level sums for sub collections like storage.total + */ + foreach ($subCollectionTotals as $subCollection => $count) { + $dbForProject->setNamespace("_project_{$projectId}"); + + $metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + } + } catch (\Exception$e) { + Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } - } while (!empty($projects)); - } + } + } while (!empty($projects)); $iterations++; $loopTook = microtime(true) - $loopStart; From e479df276fb80ee66174e101a2c7950917761014 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 20 Feb 2022 09:38:52 +0000 Subject: [PATCH 4/6] more console info --- app/tasks/usage.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index a81e5bbc20..e308aecbe5 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -370,18 +370,22 @@ $cli } } - /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ if ($iterations % 30 != 0) { // Aggregate aggregate number of objects in database only after 15 minutes $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); return; - } - $latestProject = null; + } + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating database counters."); + + $latestProject = null; do { // Loop over all the projects $attempts = 0; $max = 10; From 03c94627adb51b67e06757464e5d4d4a4990c845 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 20 Feb 2022 10:03:19 +0000 Subject: [PATCH 5/6] seems client check is not required --- app/tasks/usage.php | 160 ++++++++++++++++++++++---------------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index e308aecbe5..3b4d66fb94 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -236,6 +236,8 @@ $cli $max = 10; $sleep = 1; + $db = null; + $redis = null; do { // connect to db try { $attempts++; @@ -275,97 +277,95 @@ $cli * @var InfluxDB\Client $client */ $client = $register->get('influxdb'); - if ($client) { - $attempts = 0; - $max = 10; - $sleep = 1; + $attempts = 0; + $max = 10; + $sleep = 1; - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if(in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable $th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if(in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful } - } while ($attempts < $max); + } catch (\Throwable $th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); - // sync data - foreach ($globalMetrics as $metric => $options) { //for each metrics - foreach ($periods as $period) { // aggregate data for each period - $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); - if (!empty($latestTime[$metric][$period['key']])) { - $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); - } - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + // sync data + foreach ($globalMetrics as $metric => $options) { //for each metrics + foreach ($periods as $period) { // aggregate data for each period + $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + if (!empty($latestTime[$metric][$period['key']])) { + $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $table = $options['table']; //Which influxdb table to query for this metric - $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + $table = $options['table']; //Which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status - if (!empty($filters)) { - $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters))); - } else { - $filters = ''; - } + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters))); + } else { + $filters = ''; + } - $query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)"; - try { - $result = $database->query($query); - - $points = $result->getPoints(); - foreach ($points as $point) { - $projectId = $point['projectId']; - - if (!empty($projectId) && $projectId !== 'console') { - $dbForProject->setNamespace('_project_' . $projectId); - $metricUpdated = $metric; - - if (!empty($groupBy)) { - $groupedBy = $point[$options['groupBy']] ?? ''; - if (empty($groupedBy)) { - continue; - } - $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); + $query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)"; + try { + $result = $database->query($query); + + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + + if (!empty($projectId) && $projectId !== 'console') { + $dbForProject->setNamespace('_project_' . $projectId); + $metricUpdated = $metric; + + if (!empty($groupBy)) { + $groupedBy = $point[$options['groupBy']] ?? ''; + if (empty($groupedBy)) { + continue; } - - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric - $value = (!empty($point['value'])) ? $point['value'] : 0; - - try { - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period['key'], - 'time' => $time, - 'metric' => $metricUpdated, - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - $latestTime[$metric][$period['key']] = $time; - } catch (\Exception $e) { // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); + $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); + } + + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric + $value = (!empty($point['value'])) ? $point['value'] : 0; + + try { + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period['key'], + 'time' => $time, + 'metric' => $metricUpdated, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); } + $latestTime[$metric][$period['key']] = $time; + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); } } - } catch (\Exception $e) { - Console::warning("Failed to Query: {$e->getMessage()}"); } + } catch (\Exception $e) { + Console::warning("Failed to Query: {$e->getMessage()}"); } } } From 7d36414889b55ef5b72750f316a2f893e8cfa80e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 20 Feb 2022 10:17:57 +0000 Subject: [PATCH 6/6] fix documents table name --- app/tasks/usage.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 3b4d66fb94..0ee28f5cb6 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -467,6 +467,7 @@ $cli 'namespace' => '', 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting 'documents' => [ + 'collectionPrefix' => 'collection_', 'namespace' => '', ], ],