From 5e6b8ec50695197aae53e791cdea32c0673e7dc1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 23 Sep 2022 05:51:26 +0000 Subject: [PATCH] refactor counters to get sum in period only --- src/Appwrite/Usage/Calculators/TimeSeries.php | 181 ++++-------------- 1 file changed, 37 insertions(+), 144 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 54aff2d9f8..ebe49aca63 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -14,7 +14,20 @@ class TimeSeries extends Calculator protected Database $database; protected $errorHandler; private array $latestTime = []; - + protected array $periods = [ + [ + 'key' => '1h', + 'startTime' => '-24 hours' + ], + [ + 'key' => '1d', + 'startTime' => '-30 days' + ], + [ + 'key' => '1mo', + 'startTime' => '-12 months' + ] + ]; // all the mertics that we are collecting protected array $metrics = [ 'project.$all.network.requests' => [ @@ -271,19 +284,28 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_project_{scope}_compute_time', 'groupBy' => ['functionId'], ], - ]; - - private array $counterMetrics = [ + + // counters 'users.$all.count.total' => [ 'table' => 'appwrite_usage_users_{scope}_count_total', ], + 'buckets.$all.count.total' => [ + 'table' => 'appwrite_usage_buckets_{scope}_count_total', + ], + 'files.$all.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + ], + 'files.bucketId.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + 'groupBy' => ['bucketId'] + ], 'databases.$all.count.total' => [ 'table' => 'appwrite_usage_databases_{scope}_count_total', ], 'collections.$all.count.total' => [ 'table' => 'appwrite_usage_collections_{scope}_count_total', ], - 'documents.$all.count.total' => [ + 'documents.databaseId.count.total' => [ 'table' => 'appwrite_usage_documents_{scope}_count_total', 'groupBy' => ['databaseId'] ], @@ -301,11 +323,6 @@ class TimeSeries extends Calculator ], ]; - protected array $period = [ - 'key' => '1h', - 'startTime' => '-24 hours', - ]; - public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) { $this->database = $database; @@ -359,121 +376,6 @@ class TimeSeries extends Calculator } } } - /** - * Create or Update Count Mertic - * Create or update each metric in the stats collection for the given project - * - * @param string $projectId - * @param int $time - * @param string $period - * @param string $metric - * @param int $value - * @param int $type - * - * @return void - */ - private function createOrUpdateCountMetric(string $projectId, string $time, string $period, string $metric, int $value): void - { - $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_console'); - $project = $this->database->getDocument('projects', $projectId); - $this->database->setNamespace('_' . $project->getInternalId()); - - try { - $document = $this->database->getDocument('stats', $id); - if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 1, - ])); - } else { - $current = $document->getAttribute('value', 0); - $value = $current + $value; - $this->database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } - } - - private function syncCounters(string $metric, array $options, array $period) { - $start = null; // need to calculate last updated date - if (!empty($this->latestTime[$metric][$period['key']])) { - $start = $this->latestTime[$metric][$period['key']]; - } - $end = (new DateTime())->format(DateTime::RFC3339); - - $table = $options['table']; - $groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - - $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status - if (!empty($filters)) { - $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters))); - } else { - $filters = ''; - } - - $query = "SELECT sum(value) AS \"value\" "; - $query .= "FROM \"{$table}\" "; - $query .= "WHERE "; - $query .= is_null($start) ? '' : "\"time\" > '{$start}' "; - $query .= is_null($start) ? '' : "AND "; - $query .= "\"time\" < '{$end}' "; - $query .= "AND \"metric_type\"='counter' {$filters} "; - $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; - $query .= "FILL(null)"; - - try { - $result = $this->influxDB->query($query); - $points = $result->getPoints(); - foreach ($points as $point) { - $projectId = $point['projectId']; - - if (!empty($projectId) && $projectId !== 'console') { - $metricUpdated = $metric; - if (!empty($groupBy)) { - foreach ($options['groupBy'] as $groupBy) { - $groupedBy = $point[$groupBy] ?? ''; - if (empty($groupedBy)) { - continue; - } - $metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated); - } - } - - $value = (!empty($point['value'])) ? $point['value'] : 0; - - $this->createOrUpdateCountMetric( - $projectId, - $point['time'], - $period['key'], - $metricUpdated, - $value, - 0 - ); - $this->latestTime[$metric][$period['key']] = $point['time']; - } - } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb"); - } else { - throw $e; - } - } - } /** * Sync From InfluxDB @@ -559,28 +461,19 @@ class TimeSeries extends Calculator */ public function collect(): void { - foreach($this->counterMetrics as $metric => $options) { - try { - $this->syncCounters($metric, $options, $this->period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; + foreach ($this->periods as $period) { + foreach ($this->metrics as $metric => $options) { //for each metrics + try { + $this->syncFromInfluxDB($metric, $options, $period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; + } } } } - foreach ($this->metrics as $metric => $options) { //for each metrics - try { - $this->syncFromInfluxDB($metric, $options, $this->period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; - } - } - } } }