From 80502ff937d021b02dbefe818e1160b79efe9f8a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 22 Sep 2022 07:34:11 +0000 Subject: [PATCH] collect counter data --- src/Appwrite/Usage/Calculators/TimeSeries.php | 157 +++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 01c8661206..54aff2d9f8 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -273,8 +273,36 @@ class TimeSeries extends Calculator ], ]; + private array $counterMetrics = [ + 'users.$all.count.total' => [ + 'table' => 'appwrite_usage_users_{scope}_count_total', + ], + 'databases.$all.count.total' => [ + 'table' => 'appwrite_usage_databases_{scope}_count_total', + ], + 'collections.$all.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + ], + 'documents.$all.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'collections.databaseId.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId/collectionId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId', 'collectionId'] + ], + ]; + protected array $period = [ - 'key' => '30m', + 'key' => '1h', 'startTime' => '-24 hours', ]; @@ -331,6 +359,121 @@ class TimeSeries extends Calculator } } } + /** + * Create or Update Count Mertic + * Create or update each metric in the stats collection for the given project + * + * @param string $projectId + * @param int $time + * @param string $period + * @param string $metric + * @param int $value + * @param int $type + * + * @return void + */ + private function createOrUpdateCountMetric(string $projectId, string $time, string $period, string $metric, int $value): void + { + $id = \md5("{$time}_{$period}_{$metric}"); + $this->database->setNamespace('_console'); + $project = $this->database->getDocument('projects', $projectId); + $this->database->setNamespace('_' . $project->getInternalId()); + + try { + $document = $this->database->getDocument('stats', $id); + if ($document->isEmpty()) { + $this->database->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 1, + ])); + } else { + $current = $document->getAttribute('value', 0); + $value = $current + $value; + $this->database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } + } + } + + private function syncCounters(string $metric, array $options, array $period) { + $start = null; // need to calculate last updated date + if (!empty($this->latestTime[$metric][$period['key']])) { + $start = $this->latestTime[$metric][$period['key']]; + } + $end = (new DateTime())->format(DateTime::RFC3339); + + $table = $options['table']; + $groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters))); + } else { + $filters = ''; + } + + $query = "SELECT sum(value) AS \"value\" "; + $query .= "FROM \"{$table}\" "; + $query .= "WHERE "; + $query .= is_null($start) ? '' : "\"time\" > '{$start}' "; + $query .= is_null($start) ? '' : "AND "; + $query .= "\"time\" < '{$end}' "; + $query .= "AND \"metric_type\"='counter' {$filters} "; + $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; + $query .= "FILL(null)"; + + try { + $result = $this->influxDB->query($query); + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + + if (!empty($projectId) && $projectId !== 'console') { + $metricUpdated = $metric; + if (!empty($groupBy)) { + foreach ($options['groupBy'] as $groupBy) { + $groupedBy = $point[$groupBy] ?? ''; + if (empty($groupedBy)) { + continue; + } + $metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated); + } + } + + $value = (!empty($point['value'])) ? $point['value'] : 0; + + $this->createOrUpdateCountMetric( + $projectId, + $point['time'], + $period['key'], + $metricUpdated, + $value, + 0 + ); + $this->latestTime[$metric][$period['key']] = $point['time']; + } + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb"); + } else { + throw $e; + } + } + } /** * Sync From InfluxDB @@ -416,6 +559,18 @@ class TimeSeries extends Calculator */ public function collect(): void { + foreach($this->counterMetrics as $metric => $options) { + try { + $this->syncCounters($metric, $options, $this->period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; + } + } + } + foreach ($this->metrics as $metric => $options) { //for each metrics try { $this->syncFromInfluxDB($metric, $options, $this->period);