From da533571b3b4fc5448f21ab46eb9b3ae6099f206 Mon Sep 17 00:00:00 2001
From: Damodar Lohani <dlohani48@gmail.com>
Date: Mon, 13 Jun 2022 16:41:24 +0545
Subject: [PATCH 1/7] usage refactor initial

---
 app/tasks/usage.php            | 804 ++++++---------------------------
 src/Appwrite/Stats/Usage.php   | 273 +++++++++++
 src/Appwrite/Stats/UsageDB.php |  97 ++++
 3 files changed, 518 insertions(+), 656 deletions(-)
 create mode 100644 src/Appwrite/Stats/Usage.php
 create mode 100644 src/Appwrite/Stats/UsageDB.php

diff --git a/app/tasks/usage.php b/app/tasks/usage.php
index 829914a706..79d77816eb 100644
--- a/app/tasks/usage.php
+++ b/app/tasks/usage.php
@@ -2,14 +2,76 @@
 
 global $cli, $register;
 
+use Appwrite\Stats\Usage;
+use Appwrite\Stats\UsageDB;
+use InfluxDB\Database as InfluxDatabase;
 use Utopia\App;
-use Utopia\Cache\Adapter\Redis;
+use Utopia\Cache\Adapter\Redis as RedisCache;
 use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
+use Utopia\Registry\Registry;
+
+function getDatabase(Registry&$register, string $namespace): Database
+{
+    $attempts = 0;
+
+    do {
+        try {
+            $attempts++;
+
+            $db = $register->get('db');
+            $redis = $register->get('cache');
+
+            $cache = new Cache(new RedisCache($redis));
+            $database = new Database(new MariaDB($db), $cache);
+            $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
+            $database->setNamespace($namespace);
+
+            if (!$database->exists($database->getDefaultDatabase(), 'realtime')) {
+                throw new Exception('Collection not ready');
+            }
+            break; // leave loop if successful
+        } catch (\Exception$e) {
+            Console::warning("Database not ready. Retrying connection ({$attempts})...");
+            if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
+                throw new \Exception('Failed to connect to database: ' . $e->getMessage());
+            }
+            sleep(DATABASE_RECONNECT_SLEEP);
+        }
+    } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
+
+    return $database;
+}
+
+function getInfluxDB(Registry&$register): InfluxDatabase
+{
+    /** @var InfluxDB\Client $client */
+    $client = $register->get('influxdb');
+    $attempts = 0;
+    $max = 10;
+    $sleep = 1;
+
+    do { // check if telegraf database is ready
+        try {
+            $attempts++;
+            $database = $client->selectDB('telegraf');
+            if (in_array('telegraf', $client->listDatabases())) {
+                break; // leave the do-while if successful
+            }
+        } catch (\Throwable$th) {
+            Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
+            if ($attempts >= $max) {
+                throw new \Exception('InfluxDB database not ready yet');
+            }
+            sleep($sleep);
+        }
+    } while ($attempts < $max);
+    return $database;
+}
 
 /**
  * Metrics We collect
@@ -36,7 +98,7 @@ use Utopia\Database\Validator\Authorization;
  * database.collections.{collectionId}.documents.delete
  *
  * Storage
- * 
+ *
  * storage.buckets.create
  * storage.buckets.read
  * storage.buckets.update
@@ -90,283 +152,35 @@ $cli
         Console::success(APP_NAME . ' usage aggregation process v1 has started');
 
         $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
-        $periods = [
-            [
-                'key' => '30m',
-                'startTime' => '-24 hours',
-            ],
-            [
-                'key' => '1d',
-                'startTime' => '-90 days',
-            ],
-        ];
 
-        // all the metrics that we are collecting at the moment
-        $globalMetrics = [
-            'requests' => [
-                'table' => 'appwrite_usage_requests_all',
-            ],
-            'network' => [
-                'table' => 'appwrite_usage_network_all',
-            ],
-            'executions' => [
-                'table' => 'appwrite_usage_executions_all',
-            ],
-            'database.collections.create' => [
-                'table' => 'appwrite_usage_database_collections_create',
-            ],
-            'database.collections.read' => [
-                'table' => 'appwrite_usage_database_collections_read',
-            ],
-            'database.collections.update' => [
-                'table' => 'appwrite_usage_database_collections_update',
-            ],
-            'database.collections.delete' => [
-                'table' => 'appwrite_usage_database_collections_delete',
-            ],
-            'database.documents.create' => [
-                'table' => 'appwrite_usage_database_documents_create',
-            ],
-            'database.documents.read' => [
-                'table' => 'appwrite_usage_database_documents_read',
-            ],
-            'database.documents.update' => [
-                'table' => 'appwrite_usage_database_documents_update',
-            ],
-            'database.documents.delete' => [
-                'table' => 'appwrite_usage_database_documents_delete',
-            ],
-            'database.collections.collectionId.documents.create' => [
-                'table' => 'appwrite_usage_database_documents_create',
-                'groupBy' => 'collectionId',
-            ],
-            'database.collections.collectionId.documents.read' => [
-                'table' => 'appwrite_usage_database_documents_read',
-                'groupBy' => 'collectionId',
-            ],
-            'database.collections.collectionId.documents.update' => [
-                'table' => 'appwrite_usage_database_documents_update',
-                'groupBy' => 'collectionId',
-            ],
-            'database.collections.collectionId.documents.delete' => [
-                'table' => 'appwrite_usage_database_documents_delete',
-                'groupBy' => 'collectionId',
-            ],
-            'storage.buckets.create' => [
-                'table' => 'appwrite_usage_storage_buckets_create',
-            ],
-            'storage.buckets.read' => [
-                'table' => 'appwrite_usage_storage_buckets_read',
-            ],
-            'storage.buckets.update' => [
-                'table' => 'appwrite_usage_storage_buckets_update',
-            ],
-            'storage.buckets.delete' => [
-                'table' => 'appwrite_usage_storage_buckets_delete',
-            ],
-            'storage.files.create' => [
-                'table' => 'appwrite_usage_storage_files_create',
-            ],
-            'storage.files.read' => [
-                'table' => 'appwrite_usage_storage_files_read',
-            ],
-            'storage.files.update' => [
-                'table' => 'appwrite_usage_storage_files_update',
-            ],
-            'storage.files.delete' => [
-                'table' => 'appwrite_usage_storage_files_delete',
-            ],
-            'storage.buckets.bucketId.files.create' => [
-                'table' => 'appwrite_usage_storage_files_create',
-                'groupBy' => 'bucketId',
-            ],
-            'storage.buckets.bucketId.files.read' => [
-                'table' => 'appwrite_usage_storage_files_read',
-                'groupBy' => 'bucketId',
-            ],
-            'storage.buckets.bucketId.files.update' => [
-                'table' => 'appwrite_usage_storage_files_update',
-                'groupBy' => 'bucketId',
-            ],
-            'storage.buckets.bucketId.files.delete' => [
-                'table' => 'appwrite_usage_storage_files_delete',
-                'groupBy' => 'bucketId',
-            ],
-            'users.create' => [
-                'table' => 'appwrite_usage_users_create',
-            ],
-            'users.read' => [
-                'table' => 'appwrite_usage_users_read',
-            ],
-            'users.update' => [
-                'table' => 'appwrite_usage_users_update',
-            ],
-            'users.delete' => [
-                'table' => 'appwrite_usage_users_delete',
-            ],
-            'users.sessions.create' => [
-                'table' => 'appwrite_usage_users_sessions_create',
-            ],
-            'users.sessions.provider.create' => [
-                'table' => 'appwrite_usage_users_sessions_create',
-                'groupBy' => 'provider',
-            ],
-            'users.sessions.delete' => [
-                'table' => 'appwrite_usage_users_sessions_delete',
-            ],
-            'functions.functionId.executions' => [
-                'table' => 'appwrite_usage_executions_all',
-                'groupBy' => 'functionId',
-            ],
-            'functions.functionId.compute' => [
-                'table' => 'appwrite_usage_executions_time',
-                'groupBy' => 'functionId',
-            ],
-            'functions.functionId.failures' => [
-                'table' => 'appwrite_usage_executions_all',
-                'groupBy' => 'functionId',
-                'filters' => [
-                    'functionStatus' => 'failed',
-                ],
-            ],
-        ];
+        $database = getDatabase($register, '_console');
+        $influxDB = getInfluxDB($register);
 
-        // TODO Maybe move this to the setResource method, and reuse in the http.php file
-        $attempts = 0;
-        $max = 10;
-        $sleep = 1;
-
-        $db = null;
-        $redis = null;
-        do { // connect to db
-            try {
-                $attempts++;
-                $db = $register->get('db');
-                $redis = $register->get('cache');
-                break; // leave the do-while if successful
-            } catch (\Exception $e) {
-                Console::warning("Database not ready. Retrying connection ({$attempts})...");
-                if ($attempts >= $max) {
-                    throw new \Exception('Failed to connect to database: ' . $e->getMessage());
-                }
-                sleep($sleep);
-            }
-        } while ($attempts < $max);
-
-        // TODO use inject
-        $cacheAdapter = new Cache(new Redis($redis));
-        $dbForProject = new Database(new MariaDB($db), $cacheAdapter);
-        $dbForConsole = new Database(new MariaDB($db), $cacheAdapter);
-        $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-        $dbForConsole->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-        $dbForConsole->setNamespace('_console');
+        $usage = new Usage($database, $influxDB);
+        $usageDB = new UsageDB($database);
 
         $latestTime = [];
 
         Authorization::disable();
 
         $iterations = 0;
-        Console::loop(function () use ($interval, $register, $dbForProject, $dbForConsole, $globalMetrics, $periods, &$latestTime, &$iterations) {
+        Console::loop(function () use ($interval, $database, $usage, $usageDB, &$latestTime, &$iterations) {
             $now = date('d-m-Y H:i:s', time());
             Console::info("[{$now}] Aggregating usage data every {$interval} seconds");
 
             $loopStart = microtime(true);
 
             /**
-             * Aggregate InfluxDB every 30 seconds
-             * @var InfluxDB\Client $client
-             */
-            $client = $register->get('influxdb');
-            $attempts = 0;
-            $max = 10;
-            $sleep = 1;
-
-            do { // check if telegraf database is ready
-                try {
-                    $attempts++;
-                    $database = $client->selectDB('telegraf');
-                    if(in_array('telegraf', $client->listDatabases())) {
-                        break; // leave the do-while if successful
-                    }
-                } catch (\Throwable $th) {
-                    Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
-                    if ($attempts >= $max) {
-                        throw new \Exception('InfluxDB database not ready yet');
-                    }
-                    sleep($sleep);
-                }
-            } while ($attempts < $max);
+         * Aggregate InfluxDB every 30 seconds
+         */
 
             // sync data
-            foreach ($globalMetrics as $metric => $options) { //for each metrics
-                foreach ($periods as $period) { // aggregate data for each period
-                    $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
-                    if (!empty($latestTime[$metric][$period['key']])) {
-                        $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339);
-                    }
-                    $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339);
-
-                    $table = $options['table']; //Which influxdb table to query for this metric
-                    $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
-
-                    $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
-                    if (!empty($filters)) {
-                        $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
-                    } else {
-                        $filters = '';
-                    }
-
-                    $query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)";
+            foreach ($usage->getMetrics() as $metric => $options) { //for each metrics
+                foreach ($usage->getPeriods() as $period) { // aggregate data for each period
                     try {
-                        $result = $database->query($query);
-
-                        $points = $result->getPoints();
-                        foreach ($points as $point) {
-                            $projectId = $point['projectId'];
-
-                            if (!empty($projectId) && $projectId !== 'console') {
-                                $dbForProject->setNamespace('_' . $projectId);
-                                $metricUpdated = $metric;
-
-                                if (!empty($groupBy)) {
-                                    $groupedBy = $point[$options['groupBy']] ?? '';
-                                    if (empty($groupedBy)) {
-                                        continue;
-                                    }
-                                    $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric);
-                                }
-
-                                $time = \strtotime($point['time']);
-                                $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric
-                                $value = (!empty($point['value'])) ? $point['value'] : 0;
-
-                                try {
-                                    $document = $dbForProject->getDocument('stats', $id);
-                                    if ($document->isEmpty()) {
-                                        $dbForProject->createDocument('stats', new Document([
-                                            '$id' => $id,
-                                            'period' => $period['key'],
-                                            'time' => $time,
-                                            'metric' => $metricUpdated,
-                                            'value' => $value,
-                                            'type' => 0,
-                                        ]));
-                                    } else {
-                                        $dbForProject->updateDocument(
-                                            'stats',
-                                            $document->getId(),
-                                            $document->setAttribute('value', $value)
-                                        );
-                                    }
-                                    $latestTime[$metric][$period['key']] = $time;
-                                } catch (\Exception $e) { // if projects are deleted this might fail
-                                    Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}");
-                                    Console::warning($e->getTraceAsString());
-                                }
-                            }
-                        }
-                    } catch (\Exception $e) {
-                        Console::warning("Failed to Query: {$e->getMessage()}");
+                        $usage->syncFromInfluxDB($metric, $options, $period, $latestTime);
+                    } catch (\Exception$e) {
+                        Console::warning("Failed: {$e->getMessage()}");
                         Console::warning($e->getTraceAsString());
                     }
                 }
@@ -381,425 +195,103 @@ $cli
             }
 
             /**
-             * Aggregate MariaDB every 15 minutes
-             * Some of the queries here might contain full-table scans.
-             */
+         * Aggregate MariaDB every 15 minutes
+         * Some of the queries here might contain full-table scans.
+         */
             $now = date('d-m-Y H:i:s', time());
             Console::info("[{$now}] Aggregating database counters.");
-            
-            $latestProject = null;
-            do { // Loop over all the projects
-                $attempts = 0;
-                $max = 10;
-                $sleep = 1;
+            $usageDB->foreachDocument('console', 'projects', [], function ($project) use ($usageDB) {
+                $projectId = $project->getId();
 
-                do { // list projects
-                    try {
-                        $attempts++;
-                        $projects = $dbForConsole->find('projects', [], 100, cursor: $latestProject);
-                        break; // leave the do-while if successful
-                    } catch (\Exception $e) {
-                        Console::warning("Console DB not ready yet. Retrying ({$attempts})...");
-                        if ($attempts >= $max) {
-                            throw new \Exception('Failed access console db: ' . $e->getMessage());
-                        }
-                        sleep($sleep);
-                    }
-                } while ($attempts < $max);
-
-                if (empty($projects)) {
-                    continue;
+                // Get total storage of deployments
+                try {
+                    $deploymentsTotal = $usageDB->sum($projectId, 'deployments', 'size', 'storage.deployments.total');
+                } catch (\Exception$e) {
+                    Console::warning("Failed to save data for project {$projectId} and metric storage.deployments.total: {$e->getMessage()}");
+                    Console::warning($e->getTraceAsString());
                 }
 
-                $latestProject = $projects[array_key_last($projects)];
-
-                foreach ($projects as $project) {
-                    $projectId = $project->getId();
-
-                    // Get total storage
-                    $dbForProject->setNamespace('_' . $projectId);
-                    $deploymentsTotal = $dbForProject->sum('deployments', 'size');
-
-                    $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                    $id = \md5($time . '_30m_storage.deployments.total'); //Construct unique id for each metric using time, period and metric
-                    $document = $dbForProject->getDocument('stats', $id);
+                foreach ($usageDB->getCollections() as $collection => $options) {
                     try {
 
-                        if ($document->isEmpty()) {
-                            $dbForProject->createDocument('stats', new Document([
-                                '$id' => $id,
-                                'period' => '30m',
-                                'time' => $time,
-                                'metric' => 'storage.deployments.total',
-                                'value' => $deploymentsTotal,
-                                'type' => 1,
-                            ]));
-                        } else {
-                            $dbForProject->updateDocument(
-                                'stats',
-                                $document->getId(),
-                                $document->setAttribute('value', $deploymentsTotal)
-                            );
+                        $metricPrefix = $options['metricPrefix'] ?? '';
+                        $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
+                        $usageDB->count($projectId, $collection, $metric);
+
+                        $subCollections = $options['subCollections'] ?? [];
+
+                        if (empty($subCollections)) {
+                            continue;
                         }
-                        $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                        $id = \md5($time . '_1d_storage.deployments.total'); //Construct unique id for each metric using time, period and metric
-                        $document = $dbForProject->getDocument('stats', $id);
-                        if ($document->isEmpty()) {
-                            $dbForProject->createDocument('stats', new Document([
-                                '$id' => $id,
-                                'period' => '1d',
-                                'time' => $time,
-                                'metric' => 'storage.deployments.total',
-                                'value' => $deploymentsTotal,
-                                'type' => 1,
-                            ]));
-                        } else {
-                            $dbForProject->updateDocument(
-                                'stats',
-                                $document->getId(),
-                                $document->setAttribute('value', $deploymentsTotal)
-                            );
-                        }
-                    } catch(\Exception $e) {
-                        Console::warning("Failed to save data for project {$projectId} and metric storage.deployments.total: {$e->getMessage()}");
-                        Console::warning($e->getTraceAsString());
-                    }
 
+                        $subCollectionCounts = []; //total project level count of sub collections
+                        $subCollectionTotals = []; //total project level sum of sub collections
 
-                    $collections = [
-                        'users' => [
-                            'namespace' => '',
-                        ],
-                        'collections' => [
-                            'metricPrefix' => 'database',
-                            'namespace' => '',
-                            'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
-                                'documents' => [
-                                    'collectionPrefix' => 'collection_',
-                                    'namespace' => '',
-                                ],
-                            ],
-                        ],
-                        'buckets' => [
-                            'metricPrefix' => 'storage',
-                            'namespace' => '',
-                            'subCollections' => [
-                                'files' => [
-                                    'namespace' => '',
-                                    'collectionPrefix' => 'bucket_',
-                                    'total' => [
-                                        'field' => 'sizeOriginal'
-                                    ]
-                                ],
-                            ]
-                        ]
-                    ];
+                        $usageDB->foreachDocument($projectId, $collection, [], function ($parent) use (&$subCollectionCounts, &$subCollectionTotals, $subCollections, $projectId, $usageDB, $collection) {
+                            foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
 
-                    foreach ($collections as $collection => $options) {
-                        try {
-                            $dbForProject->setNamespace("_{$projectId}");
-                            $count = $dbForProject->count($collection);
-                            $metricPrefix = $options['metricPrefix'] ?? '';
-                            $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
+                                $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getInternalId()}.{$subCollection}.count";
 
-                            $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                            $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
-                            $document = $dbForProject->getDocument('stats', $id);
-                            if ($document->isEmpty()) {
-                                $dbForProject->createDocument('stats', new Document([
-                                    '$id' => $id,
-                                    'time' => $time,
-                                    'period' => '30m',
-                                    'metric' => $metric,
-                                    'value' => $count,
-                                    'type' => 1,
-                                ]));
-                            } else {
-                                $dbForProject->updateDocument(
-                                    'stats',
-                                    $document->getId(),
-                                    $document->setAttribute('value', $count)
-                                );
-                            }
+                                $count = $usageDB->count($projectId, ($subOptions['collectionPrefix'] ?? '') . $parent->getInternalId(), $metric);
 
-                            $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                            $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
-                            $document = $dbForProject->getDocument('stats', $id);
-                            if ($document->isEmpty()) {
-                                $dbForProject->createDocument('stats', new Document([
-                                    '$id' => $id,
-                                    'time' => $time,
-                                    'period' => '1d',
-                                    'metric' => $metric,
-                                    'value' => $count,
-                                    'type' => 1,
-                                ]));
-                            } else {
-                                $dbForProject->updateDocument(
-                                    'stats',
-                                    $document->getId(),
-                                    $document->setAttribute('value', $count)
-                                );
-                            }
+                                $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
 
-                            $subCollections = $options['subCollections'] ?? [];
-
-                            if (empty($subCollections)) {
-                                continue;
-                            }
-
-                            $latestParent = null;
-                            $subCollectionCounts = []; //total project level count of sub collections
-                            $subCollectionTotals = []; //total project level sum of sub collections
-
-                            do { // Loop over all the parent collection document for each sub collection
-                                $dbForProject->setNamespace("_{$projectId}");
-                                $parents = $dbForProject->find($collection, [], 100, cursor: $latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections
-
-                                if (empty($parents)) {
+                                // check if sum calculation is required
+                                $total = $subOptions['total'] ?? [];
+                                if (empty($total)) {
                                     continue;
                                 }
 
-                                $latestParent = $parents[array_key_last($parents)];
+                                $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getInternalId()}.{$subCollection}.total";
+                                $total = $usageDB->sum($projectId, ($subOptions['collectionPrefix'] ?? '') . $parent->getInternalId(), $total['field'], $metric);
 
-                                foreach ($parents as $parent) {
-                                    foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
-                                        $dbForProject->setNamespace("_{$projectId}");
-                                        $count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getInternalId());
-
-                                        $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
-
-                                        $dbForProject->setNamespace("_{$projectId}");
-
-                                        $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getInternalId()}.{$subCollection}.count";
-                                        $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                                        $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
-                                        $document = $dbForProject->getDocument('stats', $id);
-                                        if ($document->isEmpty()) {
-                                            $dbForProject->createDocument('stats', new Document([
-                                                '$id' => $id,
-                                                'time' => $time,
-                                                'period' => '30m',
-                                                'metric' => $metric,
-                                                'value' => $count,
-                                                'type' => 1,
-                                            ]));
-                                        } else {
-                                            $dbForProject->updateDocument(
-                                                'stats',
-                                                $document->getId(),
-                                                $document->setAttribute('value', $count)
-                                            );
-                                        }
-
-                                        $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                                        $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
-                                        $document = $dbForProject->getDocument('stats', $id);
-                                        if ($document->isEmpty()) {
-                                            $dbForProject->createDocument('stats', new Document([
-                                                '$id' => $id,
-                                                'time' => $time,
-                                                'period' => '1d',
-                                                'metric' => $metric,
-                                                'value' => $count,
-                                                'type' => 1,
-                                            ]));
-                                        } else {
-                                            $dbForProject->updateDocument(
-                                                'stats',
-                                                $document->getId(),
-                                                $document->setAttribute('value', $count)
-                                            );
-                                        }
-
-                                        // check if sum calculation is required
-                                        $total = $subOptions['total'] ?? [];
-                                        if(empty($total)) {
-                                            continue;
-                                        }
-
-                                        $dbForProject->setNamespace("_{$projectId}");
-                                        $total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getInternalId(), $total['field']);
-
-                                        $subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total
-
-                                        $dbForProject->setNamespace("_{$projectId}");
-
-                                        $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getInternalId()}.{$subCollection}.total";
-                                        $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                                        $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
-                                        $document = $dbForProject->getDocument('stats', $id);
-                                        if ($document->isEmpty()) {
-                                            $dbForProject->createDocument('stats', new Document([
-                                                '$id' => $id,
-                                                'time' => $time,
-                                                'period' => '30m',
-                                                'metric' => $metric,
-                                                'value' => $total,
-                                                'type' => 1,
-                                            ]));
-                                        } else {
-                                            $dbForProject->updateDocument('stats', $document->getId(),
-                                            $document->setAttribute('value', $total));
-                                        }
-
-                                        $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                                        $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
-                                        $document = $dbForProject->getDocument('stats', $id);
-                                        if ($document->isEmpty()) {
-                                            $dbForProject->createDocument('stats', new Document([
-                                                '$id' => $id,
-                                                'time' => $time,
-                                                'period' => '1d',
-                                                'metric' => $metric,
-                                                'value' => $total,
-                                                'type' => 1,
-                                            ]));
-                                        } else {
-                                            $dbForProject->updateDocument('stats', $document->getId(),
-                                            $document->setAttribute('value', $total));
-                                        }
-
-                                    }
-                                }
-                            } while (!empty($parents));
-
-                            /**
-                             * Inserting project level counts for sub collections like database.documents.count
-                             */
-                            foreach ($subCollectionCounts as $subCollection => $count) {
-                                $dbForProject->setNamespace("_{$projectId}");
-
-                                $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
-
-                                $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                                $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
-                                $document = $dbForProject->getDocument('stats', $id);
-                                if ($document->isEmpty()) {
-                                    $dbForProject->createDocument('stats', new Document([
-                                        '$id' => $id,
-                                        'time' => $time,
-                                        'period' => '30m',
-                                        'metric' => $metric,
-                                        'value' => $count,
-                                        'type' => 1,
-                                    ]));
-                                } else {
-                                    $dbForProject->updateDocument(
-                                        'stats',
-                                        $document->getId(),
-                                        $document->setAttribute('value', $count)
-                                    );
-                                }
-
-                                $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                                $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
-                                $document = $dbForProject->getDocument('stats', $id);
-                                if ($document->isEmpty()) {
-                                    $dbForProject->createDocument('stats', new Document([
-                                        '$id' => $id,
-                                        'time' => $time,
-                                        'period' => '1d',
-                                        'metric' => $metric,
-                                        'value' => $count,
-                                        'type' => 1,
-                                    ]));
-                                } else {
-                                    $dbForProject->updateDocument(
-                                        'stats',
-                                        $document->getId(),
-                                        $document->setAttribute('value', $count)
-                                    );
-                                }
+                                $subCollectionTotals[$subCollection] = ($subCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total
                             }
+                        });
 
-                            /**
-                             * Inserting project level sums for sub collections like storage.files.total
-                             */
-                            foreach ($subCollectionTotals as $subCollection => $count) {
-                                $dbForProject->setNamespace("_{$projectId}");
+                        /**
+                     * Inserting project level counts for sub collections like database.documents.count
+                     */
+                        foreach ($subCollectionCounts as $subCollection => $count) {
 
-                                $metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total";
+                            $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
 
-                                $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                                $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
-                                $document = $dbForProject->getDocument('stats', $id);
-                                if ($document->isEmpty()) {
-                                    $dbForProject->createDocument('stats', new Document([
-                                        '$id' => $id,
-                                        'time' => $time,
-                                        'period' => '30m',
-                                        'metric' => $metric,
-                                        'value' => $count,
-                                        'type' => 1,
-                                    ]));
-                                } else {
-                                    $dbForProject->updateDocument('stats', $document->getId(),
-                                    $document->setAttribute('value', $count));
-                                }
+                            $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
+                            $usageDB->createOrUpdateMetric($projectId, $time, '30m', $metric, $count, 1);
 
-                                $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                                $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
-                                $document = $dbForProject->getDocument('stats', $id);
-                                if ($document->isEmpty()) {
-                                    $dbForProject->createDocument('stats', new Document([
-                                        '$id' => $id,
-                                        'time' => $time,
-                                        'period' => '1d',
-                                        'metric' => $metric,
-                                        'value' => $count,
-                                        'type' => 1,
-                                    ]));
-                                } else {
-                                    $dbForProject->updateDocument('stats', $document->getId(),
-                                    $document->setAttribute('value', $count));
-                                }
-
-                                // aggregate storage.total = storage.files.total + storage.deployments.total
-                                if($metricPrefix === 'storage' && $subCollection === 'files') {
-                                    $metric = 'storage.total';
-                                    $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                                    $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
-                                    $document = $dbForProject->getDocument('stats', $id);
-                                    if ($document->isEmpty()) {
-                                        $dbForProject->createDocument('stats', new Document([
-                                            '$id' => $id,
-                                            'time' => $time,
-                                            'period' => '30m',
-                                            'metric' => $metric,
-                                            'value' => $count + $deploymentsTotal,
-                                            'type' => 1,
-                                        ]));
-                                    } else {
-                                        $dbForProject->updateDocument('stats', $document->getId(),
-                                        $document->setAttribute('value', $count + $deploymentsTotal));
-                                    }
-
-                                    $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                                    $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
-                                    $document = $dbForProject->getDocument('stats', $id);
-                                    if ($document->isEmpty()) {
-                                        $dbForProject->createDocument('stats', new Document([
-                                            '$id' => $id,
-                                            'time' => $time,
-                                            'period' => '1d',
-                                            'metric' => $metric,
-                                            'value' => $count + $deploymentsTotal,
-                                            'type' => 1,
-                                        ]));
-                                    } else {
-                                        $dbForProject->updateDocument('stats', $document->getId(),
-                                        $document->setAttribute('value', $count + $deploymentsTotal));
-                                    }
-                                }
-                            }
-                        } catch (\Exception$e) {
-                            Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}");
-                            Console::warning($e->getTraceAsString());
+                            $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
+                            $usageDB->createOrUpdateMetric($projectId, $time, '1d', $metric, $count, 1);
                         }
+
+                        /**
+                     * Inserting project level sums for sub collections like storage.files.total
+                     */
+                        foreach ($subCollectionTotals as $subCollection => $count) {
+                            $metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total";
+
+                            $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
+                            $usageDB->createOrUpdateMetric($projectId, $time, '30m', $metric, $count, 1);
+
+                            $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
+                            $usageDB->createOrUpdateMetric($projectId, $time, '1d', $metric, $count, 1);
+
+                            // aggregate storage.total = storage.files.total + storage.deployments.total
+                            if ($metricPrefix === 'storage' && $subCollection === 'files') {
+                                $metric = 'storage.total';
+
+                                $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
+                                $usageDB->createOrUpdateMetric($projectId, $time, '30m', $metric, $count + $deploymentsTotal, 1);
+
+                                $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
+                                $usageDB->createOrUpdateMetric($projectId, $time, '1d', $metric, $count + $deploymentsTotal, 1);
+                            }
+                        }
+                    } catch (\Exception$e) {
+                        Console::warning("Failed: {$e->getMessage()}");
+                        Console::warning($e->getTraceAsString());
                     }
                 }
-            } while (!empty($projects));
+            });
 
             $iterations++;
             $loopTook = microtime(true) - $loopStart;
@@ -807,4 +299,4 @@ $cli
 
             Console::info("[{$now}] Aggregation took {$loopTook} seconds");
         }, $interval);
-    });
\ No newline at end of file
+    });
diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php
new file mode 100644
index 0000000000..2296642170
--- /dev/null
+++ b/src/Appwrite/Stats/Usage.php
@@ -0,0 +1,273 @@
+<?php
+
+namespace Appwrite\Stats;
+
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use InfluxDB\Database as InfluxDatabase;
+use DateTime;
+
+class Usage {
+    protected InfluxDatabase $influxDB;
+    protected Database $database;
+
+    // all the mertics that we are collecting
+    protected array $metrics = [
+        'requests' => [
+            'table' => 'appwrite_usage_requests_all',
+        ],
+        'network' => [
+            'table' => 'appwrite_usage_network_all',
+        ],
+        'executions' => [
+            'table' => 'appwrite_usage_executions_all',
+        ],
+        'database.collections.create' => [
+            'table' => 'appwrite_usage_database_collections_create',
+        ],
+        'database.collections.read' => [
+            'table' => 'appwrite_usage_database_collections_read',
+        ],
+        'database.collections.update' => [
+            'table' => 'appwrite_usage_database_collections_update',
+        ],
+        'database.collections.delete' => [
+            'table' => 'appwrite_usage_database_collections_delete',
+        ],
+        'database.documents.create' => [
+            'table' => 'appwrite_usage_database_documents_create',
+        ],
+        'database.documents.read' => [
+            'table' => 'appwrite_usage_database_documents_read',
+        ],
+        'database.documents.update' => [
+            'table' => 'appwrite_usage_database_documents_update',
+        ],
+        'database.documents.delete' => [
+            'table' => 'appwrite_usage_database_documents_delete',
+        ],
+        'database.collections.collectionId.documents.create' => [
+            'table' => 'appwrite_usage_database_documents_create',
+            'groupBy' => 'collectionId',
+        ],
+        'database.collections.collectionId.documents.read' => [
+            'table' => 'appwrite_usage_database_documents_read',
+            'groupBy' => 'collectionId',
+        ],
+        'database.collections.collectionId.documents.update' => [
+            'table' => 'appwrite_usage_database_documents_update',
+            'groupBy' => 'collectionId',
+        ],
+        'database.collections.collectionId.documents.delete' => [
+            'table' => 'appwrite_usage_database_documents_delete',
+            'groupBy' => 'collectionId',
+        ],
+        'storage.buckets.create' => [
+            'table' => 'appwrite_usage_storage_buckets_create',
+        ],
+        'storage.buckets.read' => [
+            'table' => 'appwrite_usage_storage_buckets_read',
+        ],
+        'storage.buckets.update' => [
+            'table' => 'appwrite_usage_storage_buckets_update',
+        ],
+        'storage.buckets.delete' => [
+            'table' => 'appwrite_usage_storage_buckets_delete',
+        ],
+        'storage.files.create' => [
+            'table' => 'appwrite_usage_storage_files_create',
+        ],
+        'storage.files.read' => [
+            'table' => 'appwrite_usage_storage_files_read',
+        ],
+        'storage.files.update' => [
+            'table' => 'appwrite_usage_storage_files_update',
+        ],
+        'storage.files.delete' => [
+            'table' => 'appwrite_usage_storage_files_delete',
+        ],
+        'storage.buckets.bucketId.files.create' => [
+            'table' => 'appwrite_usage_storage_files_create',
+            'groupBy' => 'bucketId',
+        ],
+        'storage.buckets.bucketId.files.read' => [
+            'table' => 'appwrite_usage_storage_files_read',
+            'groupBy' => 'bucketId',
+        ],
+        'storage.buckets.bucketId.files.update' => [
+            'table' => 'appwrite_usage_storage_files_update',
+            'groupBy' => 'bucketId',
+        ],
+        'storage.buckets.bucketId.files.delete' => [
+            'table' => 'appwrite_usage_storage_files_delete',
+            'groupBy' => 'bucketId',
+        ],
+        'users.create' => [
+            'table' => 'appwrite_usage_users_create',
+        ],
+        'users.read' => [
+            'table' => 'appwrite_usage_users_read',
+        ],
+        'users.update' => [
+            'table' => 'appwrite_usage_users_update',
+        ],
+        'users.delete' => [
+            'table' => 'appwrite_usage_users_delete',
+        ],
+        'users.sessions.create' => [
+            'table' => 'appwrite_usage_users_sessions_create',
+        ],
+        'users.sessions.provider.create' => [
+            'table' => 'appwrite_usage_users_sessions_create',
+            'groupBy' => 'provider',
+        ],
+        'users.sessions.delete' => [
+            'table' => 'appwrite_usage_users_sessions_delete',
+        ],
+        'functions.functionId.executions' => [
+            'table' => 'appwrite_usage_executions_all',
+            'groupBy' => 'functionId',
+        ],
+        'functions.functionId.compute' => [
+            'table' => 'appwrite_usage_executions_time',
+            'groupBy' => 'functionId',
+        ],
+        'functions.functionId.failures' => [
+            'table' => 'appwrite_usage_executions_all',
+            'groupBy' => 'functionId',
+            'filters' => [
+                'functionStatus' => 'failed',
+            ],
+        ],
+    ];
+
+    protected array $periods = [
+        [
+            'key' => '30m',
+            'startTime' => '-24 hours',
+        ],
+        [
+            'key' => '1d',
+            'startTime' => '-90 days',
+        ],
+    ];
+
+    public function __construct(Database $database, InfluxDatabase $influxDB)
+    {
+        $this->database = $database;
+        $this->influxDB = $influxDB;
+    }
+
+    public function getMetrics(): array
+    {
+        return $this->metrics;
+    }
+
+    public function getPeriods(): array
+    {
+        return $this->periods;
+    }
+
+    /**
+     * Create or Update Mertic
+     * Create or update each metric in the stats collection for the given project
+     * 
+     * @param string $projectId
+     * @param int $time
+     * @param string $period
+     * @param string $metric
+     * @param int $value
+     * @param int $type
+     * 
+     * @return void
+     */
+    public function createOrUpdateMetric(string $projectId, int $time, string $period, string $metric, int $value, int $type): void
+    {
+        $id = \md5("{$time}_{$period}_{$metric}");
+        $this->database->setNamespace('_' . $projectId);
+        try {
+            $document = $this->database->getDocument('stats', $id);
+            if ($document->isEmpty()) {
+                $this->database->createDocument('stats', new Document([
+                    '$id' => $id,
+                    'period' => $period['key'],
+                    'time' => $time,
+                    'metric' => $metric,
+                    'value' => $value,
+                    'type' => $type,
+                ]));
+            } else {
+                $this->database->updateDocument(
+                    'stats',
+                    $document->getId(),
+                    $document->setAttribute('value', $value)
+                );
+            }
+            $latestTime[$metric][$period['key']] = $time;
+        } catch (\Exception $e) { // if projects are deleted this might fail
+            throw new \Exception("Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}");
+        }
+    }
+
+    /**
+     * Sync From InfluxDB
+     * Sync stats from influxDB to stats collection in the Appwrite database
+     * 
+     * @param string $metric
+     * @param array $options
+     * @param array $period
+     * @param array $latestTime
+     * 
+     * @return void
+     */
+    public function syncFromInfluxDB(string $metric, array $options, array $period, array &$latestTime): void
+    {
+        $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
+        if (!empty($latestTime[$metric][$period['key']])) {
+            $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339);
+        }
+        $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339);
+
+        $table = $options['table']; //Which influxdb table to query for this metric
+        $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
+
+        $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
+        if (!empty($filters)) {
+            $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
+        } else {
+            $filters = '';
+        }
+
+        $query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)";
+        $result = $this->influxDB->query($query);
+
+        $points = $result->getPoints();
+        foreach ($points as $point) {
+            $projectId = $point['projectId'];
+
+            if (!empty($projectId) && $projectId !== 'console') {
+                $metricUpdated = $metric;
+
+                if (!empty($groupBy)) {
+                    $groupedBy = $point[$options['groupBy']] ?? '';
+                    if (empty($groupedBy)) {
+                        continue;
+                    }
+                    $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric);
+                }
+
+                $time = \strtotime($point['time']);
+                $value = (!empty($point['value'])) ? $point['value'] : 0;
+
+                $this->createOrUpdateMetric(
+                    $projectId,
+                    $time,
+                    $period['key'],
+                    $metricUpdated,
+                    $value,
+                    0
+                );
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php
new file mode 100644
index 0000000000..1e2bd92acb
--- /dev/null
+++ b/src/Appwrite/Stats/UsageDB.php
@@ -0,0 +1,97 @@
+<?php
+
+namespace Appwrite\Stats;
+
+use Utopia\Database\Database;
+
+class UsageDB extends Usage
+{
+    protected array $collections = [
+        'users' => [
+            'namespace' => '',
+        ],
+        'collections' => [
+            'metricPrefix' => 'database',
+            'namespace' => '',
+            'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
+                'documents' => [
+                    'collectionPrefix' => 'collection_',
+                    'namespace' => '',
+                ],
+            ],
+        ],
+        'buckets' => [
+            'metricPrefix' => 'storage',
+            'namespace' => '',
+            'subCollections' => [
+                'files' => [
+                    'namespace' => '',
+                    'collectionPrefix' => 'bucket_',
+                    'total' => [
+                        'field' => 'sizeOriginal',
+                    ],
+                ],
+            ],
+        ],
+    ];
+
+    public function __construct(Database $database)
+    {
+        $this->database = $database;
+    }
+
+    public function getCollections(): array
+    {
+        return $this->collections;
+    }
+
+    public function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
+    {
+        $limit = 50;
+        $results = [];
+        $sum = $limit;
+        $latestDocument = null;
+        $this->database->setNamespace('_' . $projectId);
+
+        while ($sum === $limit) {
+            $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument);
+
+            $sum = count($results);
+
+            foreach ($results as $document) {
+                if (is_callable($callback)) {
+                    $callback($document);
+                }
+            }
+            $latestDocument = $results[array_key_last($results)];
+        }
+    }
+
+    public function sum(string $projectId, string $collection, string $attribute, string $metric): int
+    {
+        $this->database->setNamespace('_' . $projectId);
+        $sum = (int) $this->database->sum($collection, $attribute);
+
+        $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
+        $this->createOrUpdateMetric($projectId, $time, '30m', $metric, $sum, 1);
+
+        $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
+        $this->createOrUpdateMetric($projectId, $time, '1d', $metric, $sum, 1);
+        return $sum;
+    }
+
+    public function count(string $projectId, string $collection, string $metric): int
+    {
+        $this->database->setNamespace("_{$projectId}");
+        $count = $this->database->count($collection);
+        $metricPrefix = $options['metricPrefix'] ?? '';
+        $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
+
+        $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
+        $this->createOrUpdateMetric($projectId, $time, '30m', $metric, $count, 1);
+        
+        $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
+        $this->createOrUpdateMetric($projectId, $time, '1d', $metric, $count, 1);
+        return $count;
+    }
+}

From 2a13cc33ac3f754b893a3f0b31418cc66e7c8920 Mon Sep 17 00:00:00 2001
From: Damodar Lohani <dlohani48@gmail.com>
Date: Mon, 13 Jun 2022 16:56:26 +0545
Subject: [PATCH 2/7] cleaning up refactors

---
 app/tasks/usage.php            | 206 +++-------------------------
 src/Appwrite/Stats/Usage.php   |  57 +++++---
 src/Appwrite/Stats/UsageDB.php | 236 ++++++++++++++++++++++++++-------
 3 files changed, 245 insertions(+), 254 deletions(-)

diff --git a/app/tasks/usage.php b/app/tasks/usage.php
index 79d77816eb..6ebc35f747 100644
--- a/app/tasks/usage.php
+++ b/app/tasks/usage.php
@@ -11,7 +11,6 @@ use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Database;
-use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Registry\Registry;
 
@@ -31,8 +30,8 @@ function getDatabase(Registry&$register, string $namespace): Database
             $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
             $database->setNamespace($namespace);
 
-            if (!$database->exists($database->getDefaultDatabase(), 'realtime')) {
-                throw new Exception('Collection not ready');
+            if (!$database->exists($database->getDefaultDatabase(), 'projects')) {
+                throw new Exception('Projects collection not ready');
             }
             break; // leave loop if successful
         } catch (\Exception$e) {
@@ -73,81 +72,15 @@ function getInfluxDB(Registry&$register): InfluxDatabase
     return $database;
 }
 
-/**
- * Metrics We collect
- *
- * General
- *
- * requests
- * network
- * executions
- *
- * Database
- *
- * database.collections.create
- * database.collections.read
- * database.collections.update
- * database.collections.delete
- * database.documents.create
- * database.documents.read
- * database.documents.update
- * database.documents.delete
- * database.collections.{collectionId}.documents.create
- * database.collections.{collectionId}.documents.read
- * database.collections.{collectionId}.documents.update
- * database.collections.{collectionId}.documents.delete
- *
- * Storage
- *
- * storage.buckets.create
- * storage.buckets.read
- * storage.buckets.update
- * storage.buckets.delete
- * storage.files.create
- * storage.files.read
- * storage.files.update
- * storage.files.delete
- * storage.buckets.{bucketId}.files.create
- * storage.buckets.{bucketId}.files.read
- * storage.buckets.{bucketId}.files.update
- * storage.buckets.{bucketId}.files.delete
- *
- * Users
- *
- * users.create
- * users.read
- * users.update
- * users.delete
- * users.sessions.create
- * users.sessions.{provider}.create
- * users.sessions.delete
- *
- * Functions
- *
- * functions.{functionId}.executions
- * functions.{functionId}.failures
- * functions.{functionId}.compute
- *
- * Counters
- *
- * users.count
- * storage.buckets.count
- * storage.files.count
- * storage.buckets.{bucketId}.files.count
- * database.collections.count
- * database.documents.count
- * database.collections.{collectionId}.documents.count
- *
- * Totals
- *
- * storage.total
- *
- */
+$logError = function($message, $stackTrace) {
+    Console::warning("Failed: {$message}");
+    Console::warning($stackTrace);
+};
 
 $cli
     ->task('usage')
     ->desc('Schedules syncing data from influxdb to Appwrite console db')
-    ->action(function () use ($register) {
+    ->action(function () use ($register, $logError) {
         Console::title('Usage Aggregation V1');
         Console::success(APP_NAME . ' usage aggregation process v1 has started');
 
@@ -156,37 +89,25 @@ $cli
         $database = getDatabase($register, '_console');
         $influxDB = getInfluxDB($register);
 
-        $usage = new Usage($database, $influxDB);
-        $usageDB = new UsageDB($database);
+        $usage = new Usage($database, $influxDB, $logError);
 
-        $latestTime = [];
+        $usageDB = new UsageDB($database, $logError);
 
         Authorization::disable();
 
         $iterations = 0;
-        Console::loop(function () use ($interval, $database, $usage, $usageDB, &$latestTime, &$iterations) {
+        Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) {
             $now = date('d-m-Y H:i:s', time());
             Console::info("[{$now}] Aggregating usage data every {$interval} seconds");
 
             $loopStart = microtime(true);
 
             /**
-         * Aggregate InfluxDB every 30 seconds
-         */
+             * Aggregate InfluxDB every 30 seconds
+             */
+            $usage->collect();
 
-            // sync data
-            foreach ($usage->getMetrics() as $metric => $options) { //for each metrics
-                foreach ($usage->getPeriods() as $period) { // aggregate data for each period
-                    try {
-                        $usage->syncFromInfluxDB($metric, $options, $period, $latestTime);
-                    } catch (\Exception$e) {
-                        Console::warning("Failed: {$e->getMessage()}");
-                        Console::warning($e->getTraceAsString());
-                    }
-                }
-            }
-
-            if ($iterations % 30 != 0) { // Aggregate aggregate number of objects in database only after 15 minutes
+            if ($iterations % 30 != 0) { // return if 30 iterations has not passed
                 $iterations++;
                 $loopTook = microtime(true) - $loopStart;
                 $now = date('d-m-Y H:i:s', time());
@@ -194,104 +115,15 @@ $cli
                 return;
             }
 
+            $iterations = 0; // Reset iterations to prevent overflow when running for long time
             /**
-         * Aggregate MariaDB every 15 minutes
-         * Some of the queries here might contain full-table scans.
-         */
+             * Aggregate MariaDB every 15 minutes
+             * Some of the queries here might contain full-table scans.
+             */
             $now = date('d-m-Y H:i:s', time());
             Console::info("[{$now}] Aggregating database counters.");
-            $usageDB->foreachDocument('console', 'projects', [], function ($project) use ($usageDB) {
-                $projectId = $project->getId();
 
-                // Get total storage of deployments
-                try {
-                    $deploymentsTotal = $usageDB->sum($projectId, 'deployments', 'size', 'storage.deployments.total');
-                } catch (\Exception$e) {
-                    Console::warning("Failed to save data for project {$projectId} and metric storage.deployments.total: {$e->getMessage()}");
-                    Console::warning($e->getTraceAsString());
-                }
-
-                foreach ($usageDB->getCollections() as $collection => $options) {
-                    try {
-
-                        $metricPrefix = $options['metricPrefix'] ?? '';
-                        $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
-                        $usageDB->count($projectId, $collection, $metric);
-
-                        $subCollections = $options['subCollections'] ?? [];
-
-                        if (empty($subCollections)) {
-                            continue;
-                        }
-
-                        $subCollectionCounts = []; //total project level count of sub collections
-                        $subCollectionTotals = []; //total project level sum of sub collections
-
-                        $usageDB->foreachDocument($projectId, $collection, [], function ($parent) use (&$subCollectionCounts, &$subCollectionTotals, $subCollections, $projectId, $usageDB, $collection) {
-                            foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
-
-                                $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getInternalId()}.{$subCollection}.count";
-
-                                $count = $usageDB->count($projectId, ($subOptions['collectionPrefix'] ?? '') . $parent->getInternalId(), $metric);
-
-                                $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
-
-                                // check if sum calculation is required
-                                $total = $subOptions['total'] ?? [];
-                                if (empty($total)) {
-                                    continue;
-                                }
-
-                                $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getInternalId()}.{$subCollection}.total";
-                                $total = $usageDB->sum($projectId, ($subOptions['collectionPrefix'] ?? '') . $parent->getInternalId(), $total['field'], $metric);
-
-                                $subCollectionTotals[$subCollection] = ($subCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total
-                            }
-                        });
-
-                        /**
-                     * Inserting project level counts for sub collections like database.documents.count
-                     */
-                        foreach ($subCollectionCounts as $subCollection => $count) {
-
-                            $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
-
-                            $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                            $usageDB->createOrUpdateMetric($projectId, $time, '30m', $metric, $count, 1);
-
-                            $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                            $usageDB->createOrUpdateMetric($projectId, $time, '1d', $metric, $count, 1);
-                        }
-
-                        /**
-                     * Inserting project level sums for sub collections like storage.files.total
-                     */
-                        foreach ($subCollectionTotals as $subCollection => $count) {
-                            $metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total";
-
-                            $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                            $usageDB->createOrUpdateMetric($projectId, $time, '30m', $metric, $count, 1);
-
-                            $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                            $usageDB->createOrUpdateMetric($projectId, $time, '1d', $metric, $count, 1);
-
-                            // aggregate storage.total = storage.files.total + storage.deployments.total
-                            if ($metricPrefix === 'storage' && $subCollection === 'files') {
-                                $metric = 'storage.total';
-
-                                $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-                                $usageDB->createOrUpdateMetric($projectId, $time, '30m', $metric, $count + $deploymentsTotal, 1);
-
-                                $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-                                $usageDB->createOrUpdateMetric($projectId, $time, '1d', $metric, $count + $deploymentsTotal, 1);
-                            }
-                        }
-                    } catch (\Exception$e) {
-                        Console::warning("Failed: {$e->getMessage()}");
-                        Console::warning($e->getTraceAsString());
-                    }
-                }
-            });
+            $usageDB->collect();
 
             $iterations++;
             $loopTook = microtime(true) - $loopStart;
diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php
index 2296642170..774ef2ccb3 100644
--- a/src/Appwrite/Stats/Usage.php
+++ b/src/Appwrite/Stats/Usage.php
@@ -10,6 +10,8 @@ use DateTime;
 class Usage {
     protected InfluxDatabase $influxDB;
     protected Database $database;
+    protected $errorHandler;
+    private array $latestTime = [];
 
     // all the mertics that we are collecting
     protected array $metrics = [
@@ -144,28 +146,21 @@ class Usage {
     protected array $periods = [
         [
             'key' => '30m',
+            'multiplier' => 1800,
             'startTime' => '-24 hours',
         ],
         [
             'key' => '1d',
+            'multiplier' => 86400,
             'startTime' => '-90 days',
         ],
     ];
 
-    public function __construct(Database $database, InfluxDatabase $influxDB)
+    public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
     {
         $this->database = $database;
         $this->influxDB = $influxDB;
-    }
-
-    public function getMetrics(): array
-    {
-        return $this->metrics;
-    }
-
-    public function getPeriods(): array
-    {
-        return $this->periods;
+        $this->errorHandler = $errorHandler;
     }
 
     /**
@@ -181,7 +176,7 @@ class Usage {
      * 
      * @return void
      */
-    public function createOrUpdateMetric(string $projectId, int $time, string $period, string $metric, int $value, int $type): void
+    private function createOrUpdateMetric(string $projectId, int $time, string $period, string $metric, int $value, int $type): void
     {
         $id = \md5("{$time}_{$period}_{$metric}");
         $this->database->setNamespace('_' . $projectId);
@@ -203,9 +198,13 @@ class Usage {
                     $document->setAttribute('value', $value)
                 );
             }
-            $latestTime[$metric][$period['key']] = $time;
+            $this->latestTime[$metric][$period['key']] = $time;
         } catch (\Exception $e) { // if projects are deleted this might fail
-            throw new \Exception("Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}");
+            if(is_callable($this->errorHandler)) {
+                call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+            } else {
+                throw $e;
+            }
         }
     }
 
@@ -216,15 +215,14 @@ class Usage {
      * @param string $metric
      * @param array $options
      * @param array $period
-     * @param array $latestTime
      * 
      * @return void
      */
-    public function syncFromInfluxDB(string $metric, array $options, array $period, array &$latestTime): void
+    private function syncFromInfluxDB(string $metric, array $options, array $period): void
     {
         $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
-        if (!empty($latestTime[$metric][$period['key']])) {
-            $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339);
+        if (!empty($this->latestTime[$metric][$period['key']])) {
+            $start = DateTime::createFromFormat('U', $this->latestTime[$metric][$period['key']])->format(DateTime::RFC3339);
         }
         $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339);
 
@@ -270,4 +268,27 @@ class Usage {
             }
         }
     }
+
+    /**
+     * Collect Stats
+     * Collect all the stats from Influd DB to Database
+     * 
+     * @return void
+     */
+    public function collect(): void
+    {
+        foreach ($this->metrics as $metric => $options) { //for each metrics
+            foreach ($this->periods as $period) { // aggregate data for each period
+                try {
+                    $this->syncFromInfluxDB($metric, $options, $period);
+                } catch (\Exception $e) {
+                    if(is_callable($this->errorHandler)) {
+                        call_user_func($this->errorHandler, $e->getMessage(), $e->getTraceAsString());
+                    } else {
+                        throw $e;
+                    }
+                }
+            }
+        }
+    }
 }
\ No newline at end of file
diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php
index 1e2bd92acb..c0e8f7fa60 100644
--- a/src/Appwrite/Stats/UsageDB.php
+++ b/src/Appwrite/Stats/UsageDB.php
@@ -3,49 +3,72 @@
 namespace Appwrite\Stats;
 
 use Utopia\Database\Database;
+use Utopia\Database\Document;
 
 class UsageDB extends Usage
 {
-    protected array $collections = [
-        'users' => [
-            'namespace' => '',
-        ],
-        'collections' => [
-            'metricPrefix' => 'database',
-            'namespace' => '',
-            'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
-                'documents' => [
-                    'collectionPrefix' => 'collection_',
-                    'namespace' => '',
-                ],
-            ],
-        ],
-        'buckets' => [
-            'metricPrefix' => 'storage',
-            'namespace' => '',
-            'subCollections' => [
-                'files' => [
-                    'namespace' => '',
-                    'collectionPrefix' => 'bucket_',
-                    'total' => [
-                        'field' => 'sizeOriginal',
-                    ],
-                ],
-            ],
-        ],
-    ];
-
-    public function __construct(Database $database)
+    public function __construct(Database $database, callable $errorHandler = null)
     {
         $this->database = $database;
+        $this->errorHandler = $errorHandler;
     }
-
-    public function getCollections(): array
+    /**
+     * Create or Update Mertic
+     * Create or update each metric in the stats collection for the given project
+     *
+     * @param string $projectId
+     * @param string $metric
+     * @param int $value
+     *
+     * @return void
+     */
+    private function createOrUpdateMetric(string $projectId, string $metric, int $value): void
     {
-        return $this->collections;
+        foreach ($this->periods as $options) {
+            $period = $options['key'];
+            $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']);
+            $id = \md5("{$time}_{$period}_{$metric}");
+            $this->database->setNamespace('_' . $projectId);
+            try {
+                $document = $this->database->getDocument('stats', $id);
+                if ($document->isEmpty()) {
+                    $this->database->createDocument('stats', new Document([
+                        '$id' => $id,
+                        'period' => $period['key'],
+                        'time' => $time,
+                        'metric' => $metric,
+                        'value' => $value,
+                        'type' => 1,
+                    ]));
+                } else {
+                    $this->database->updateDocument(
+                        'stats',
+                        $document->getId(),
+                        $document->setAttribute('value', $value)
+                    );
+                } 
+            } catch (\Exception$e) { // if projects are deleted this might fail
+                if (is_callable($this->errorHandler)) {
+                    call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+                } else {
+                    throw $e; 
+                }
+            }
+        }
     }
 
-    public function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
+    /**
+     * Foreach Document
+     * Call provided callback for each document in the collection
+     * 
+     * @param string $projectId
+     * @param string $collection
+     * @param array $queries
+     * @param callable $callback
+     * 
+     * @return void
+     */
+    private function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
     {
         $limit = 50;
         $results = [];
@@ -67,31 +90,146 @@ class UsageDB extends Usage
         }
     }
 
-    public function sum(string $projectId, string $collection, string $attribute, string $metric): int
+    /**
+     * Sum
+     * Calculate sum of a attribute of documents in collection
+     * 
+     * @param string $projectId
+     * @param string $collection
+     * @param string $attribute
+     * @param string $metric
+     * 
+     * @return int
+     */
+    private function sum(string $projectId, string $collection, string $attribute, string $metric): int
     {
         $this->database->setNamespace('_' . $projectId);
         $sum = (int) $this->database->sum($collection, $attribute);
-
-        $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-        $this->createOrUpdateMetric($projectId, $time, '30m', $metric, $sum, 1);
-
-        $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-        $this->createOrUpdateMetric($projectId, $time, '1d', $metric, $sum, 1);
+        $this->createOrUpdateMetric($projectId, $metric, $sum);
         return $sum;
     }
 
-    public function count(string $projectId, string $collection, string $metric): int
+    /**
+     * Count
+     * Count number of documents in collection
+     * 
+     * @param string $projectId
+     * @param string $collection
+     * @param string $metric
+     * 
+     * @return int
+     */
+    private function count(string $projectId, string $collection, string $metric): int
     {
         $this->database->setNamespace("_{$projectId}");
         $count = $this->database->count($collection);
-        $metricPrefix = $options['metricPrefix'] ?? '';
-        $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
 
-        $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
-        $this->createOrUpdateMetric($projectId, $time, '30m', $metric, $count, 1);
-        
-        $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
-        $this->createOrUpdateMetric($projectId, $time, '1d', $metric, $count, 1);
+        $this->createOrUpdateMetric($projectId, $metric, $count);
         return $count;
     }
+
+    /**
+     * Deployments Total
+     * Total sum of storage used by deployments
+     * 
+     * @param string $projectId
+     * 
+     * @return int
+     */
+    private function deploymentsTotal(string $projectId): int
+    {
+        return $this->sum($projectId, 'deployments', 'size', 'stroage.deployments.total');
+    }
+
+    /**
+     * Users Stats
+     * Metric: users.count
+     *
+     * @param string $projectId
+     *
+     * @return void
+     */
+    private function usersStats(string $projectId): void
+    {
+        $this->count($projectId, 'users', 'users.count');
+    }
+
+    /**
+     * Storage Stats
+     * Metrics: storage.total, storage.files.total, storage.buckets.{bucketId}.files.total,
+     * storage.buckets.count, storage.files.count, storage.buckets.{bucketId}.files.count
+     * 
+     * @param string $projectId
+     * 
+     * @return void
+     */
+    private function storageStats(string $projectId): void
+    {
+        $deploymentsTotal = $this->deploymentsTotal($projectId);
+
+        $projectFilesTotal = 0;
+        $projectFilesCount = 0;
+
+        $metric = 'storage.buckets.count';
+        $this->count($projectId, 'buckets', $metric);
+
+        $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId, ) {
+            $metric = "storage.buckets.{$bucket->getId()}.files.count";
+
+            $count = $this->count($projectId, 'buckets_' . $bucket->getInternalId(), $metric);
+            $projectFilesCount += $count;
+
+            $metric = "storage.buckets.{$bucket->getId()}.files.total";
+            $sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
+            $projectFilesTotal += $sum;
+        });
+
+        $this->createOrUpdateMetric($projectId, 'storage.files.count', $projectFilesCount);
+        $this->createOrUpdateMetric($projectId, 'storage.files.total', $projectFilesTotal);
+
+        $this->createOrUpdateMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal);
+    }
+
+    /**
+     * Database Stats
+     * Collect all database stats
+     * Metrics: database.collections.count, database.collections.{collectionId}.documents.count,
+     * database.documents.count
+     * 
+     * @param string $projectId
+     * 
+     * @return void
+     */
+    private function databaseStats(string $projectId): void
+    {
+        $projectDocumentsCount = 0;
+
+        $metric = 'database.collections.count';
+        $this->count($projectId, 'collections', $metric);
+
+        $this->foreachDocument($projectId, 'collections', [], function ($collection) use (&$projectDocumentsCount, $projectId, ) {
+            $metric = "database.collections.{$collection->getId()}.documents.count";
+
+            $count = $this->count($projectId, 'collection_' . $collection->getInternalId(), $metric);
+            $projectDocumentsCount += $count;
+        });
+
+        $this->createOrUpdateMetric($projectId, 'database.documents.count', $projectDocumentsCount);
+    }
+
+    /**
+     * Collect Stats
+     * Collect all database related stats
+     * 
+     * @return void
+     */
+    public function collect(): void
+    {
+        $this->foreachDocument('console', 'projects', [], function ($project) {
+                $projectId = $project->getId();
+                $this->usersStats($projectId);
+                $this->databaseStats($projectId);
+                $this->storageStats($projectId);
+        });
+    }
 }

From 05152da1951f1dc9b5777df9b4d2b8e6aa458a7d Mon Sep 17 00:00:00 2001
From: Damodar Lohani <dlohani48@gmail.com>
Date: Mon, 13 Jun 2022 15:49:27 +0000
Subject: [PATCH 3/7] fix formatting

---
 app/tasks/usage.php            |  6 +++---
 src/Appwrite/Stats/Usage.php   | 19 ++++++++---------
 src/Appwrite/Stats/UsageDB.php | 37 ++++++++++++++++++----------------
 3 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/app/tasks/usage.php b/app/tasks/usage.php
index 6ebc35f747..a46ab739b2 100644
--- a/app/tasks/usage.php
+++ b/app/tasks/usage.php
@@ -14,7 +14,7 @@ use Utopia\Database\Database;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Registry\Registry;
 
-function getDatabase(Registry&$register, string $namespace): Database
+function getDatabase(Registry &$register, string $namespace): Database
 {
     $attempts = 0;
 
@@ -46,7 +46,7 @@ function getDatabase(Registry&$register, string $namespace): Database
     return $database;
 }
 
-function getInfluxDB(Registry&$register): InfluxDatabase
+function getInfluxDB(Registry &$register): InfluxDatabase
 {
     /** @var InfluxDB\Client $client */
     $client = $register->get('influxdb');
@@ -72,7 +72,7 @@ function getInfluxDB(Registry&$register): InfluxDatabase
     return $database;
 }
 
-$logError = function($message, $stackTrace) {
+$logError = function ($message, $stackTrace) {
     Console::warning("Failed: {$message}");
     Console::warning($stackTrace);
 };
diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php
index 774ef2ccb3..e5901ba8f9 100644
--- a/src/Appwrite/Stats/Usage.php
+++ b/src/Appwrite/Stats/Usage.php
@@ -7,7 +7,8 @@ use Utopia\Database\Document;
 use InfluxDB\Database as InfluxDatabase;
 use DateTime;
 
-class Usage {
+class Usage
+{
     protected InfluxDatabase $influxDB;
     protected Database $database;
     protected $errorHandler;
@@ -166,14 +167,14 @@ class Usage {
     /**
      * Create or Update Mertic
      * Create or update each metric in the stats collection for the given project
-     * 
+     *
      * @param string $projectId
      * @param int $time
      * @param string $period
      * @param string $metric
      * @param int $value
      * @param int $type
-     * 
+     *
      * @return void
      */
     private function createOrUpdateMetric(string $projectId, int $time, string $period, string $metric, int $value, int $type): void
@@ -200,7 +201,7 @@ class Usage {
             }
             $this->latestTime[$metric][$period['key']] = $time;
         } catch (\Exception $e) { // if projects are deleted this might fail
-            if(is_callable($this->errorHandler)) {
+            if (is_callable($this->errorHandler)) {
                 call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
             } else {
                 throw $e;
@@ -211,11 +212,11 @@ class Usage {
     /**
      * Sync From InfluxDB
      * Sync stats from influxDB to stats collection in the Appwrite database
-     * 
+     *
      * @param string $metric
      * @param array $options
      * @param array $period
-     * 
+     *
      * @return void
      */
     private function syncFromInfluxDB(string $metric, array $options, array $period): void
@@ -272,7 +273,7 @@ class Usage {
     /**
      * Collect Stats
      * Collect all the stats from Influd DB to Database
-     * 
+     *
      * @return void
      */
     public function collect(): void
@@ -282,7 +283,7 @@ class Usage {
                 try {
                     $this->syncFromInfluxDB($metric, $options, $period);
                 } catch (\Exception $e) {
-                    if(is_callable($this->errorHandler)) {
+                    if (is_callable($this->errorHandler)) {
                         call_user_func($this->errorHandler, $e->getMessage(), $e->getTraceAsString());
                     } else {
                         throw $e;
@@ -291,4 +292,4 @@ class Usage {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php
index c0e8f7fa60..7165245497 100644
--- a/src/Appwrite/Stats/UsageDB.php
+++ b/src/Appwrite/Stats/UsageDB.php
@@ -46,12 +46,12 @@ class UsageDB extends Usage
                         $document->getId(),
                         $document->setAttribute('value', $value)
                     );
-                } 
+                }
             } catch (\Exception$e) { // if projects are deleted this might fail
                 if (is_callable($this->errorHandler)) {
                     call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
                 } else {
-                    throw $e; 
+                    throw $e;
                 }
             }
         }
@@ -60,12 +60,12 @@ class UsageDB extends Usage
     /**
      * Foreach Document
      * Call provided callback for each document in the collection
-     * 
+     *
      * @param string $projectId
      * @param string $collection
      * @param array $queries
      * @param callable $callback
-     * 
+     *
      * @return void
      */
     private function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
@@ -78,6 +78,9 @@ class UsageDB extends Usage
 
         while ($sum === $limit) {
             $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument);
+            if (empty($results)) {
+                return;
+            }
 
             $sum = count($results);
 
@@ -93,12 +96,12 @@ class UsageDB extends Usage
     /**
      * Sum
      * Calculate sum of a attribute of documents in collection
-     * 
+     *
      * @param string $projectId
      * @param string $collection
      * @param string $attribute
      * @param string $metric
-     * 
+     *
      * @return int
      */
     private function sum(string $projectId, string $collection, string $attribute, string $metric): int
@@ -112,11 +115,11 @@ class UsageDB extends Usage
     /**
      * Count
      * Count number of documents in collection
-     * 
+     *
      * @param string $projectId
      * @param string $collection
      * @param string $metric
-     * 
+     *
      * @return int
      */
     private function count(string $projectId, string $collection, string $metric): int
@@ -131,9 +134,9 @@ class UsageDB extends Usage
     /**
      * Deployments Total
      * Total sum of storage used by deployments
-     * 
+     *
      * @param string $projectId
-     * 
+     *
      * @return int
      */
     private function deploymentsTotal(string $projectId): int
@@ -158,9 +161,9 @@ class UsageDB extends Usage
      * Storage Stats
      * Metrics: storage.total, storage.files.total, storage.buckets.{bucketId}.files.total,
      * storage.buckets.count, storage.files.count, storage.buckets.{bucketId}.files.count
-     * 
+     *
      * @param string $projectId
-     * 
+     *
      * @return void
      */
     private function storageStats(string $projectId): void
@@ -173,7 +176,7 @@ class UsageDB extends Usage
         $metric = 'storage.buckets.count';
         $this->count($projectId, 'buckets', $metric);
 
-        $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId, ) {
+        $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
             $metric = "storage.buckets.{$bucket->getId()}.files.count";
 
             $count = $this->count($projectId, 'buckets_' . $bucket->getInternalId(), $metric);
@@ -195,9 +198,9 @@ class UsageDB extends Usage
      * Collect all database stats
      * Metrics: database.collections.count, database.collections.{collectionId}.documents.count,
      * database.documents.count
-     * 
+     *
      * @param string $projectId
-     * 
+     *
      * @return void
      */
     private function databaseStats(string $projectId): void
@@ -207,7 +210,7 @@ class UsageDB extends Usage
         $metric = 'database.collections.count';
         $this->count($projectId, 'collections', $metric);
 
-        $this->foreachDocument($projectId, 'collections', [], function ($collection) use (&$projectDocumentsCount, $projectId, ) {
+        $this->foreachDocument($projectId, 'collections', [], function ($collection) use (&$projectDocumentsCount, $projectId,) {
             $metric = "database.collections.{$collection->getId()}.documents.count";
 
             $count = $this->count($projectId, 'collection_' . $collection->getInternalId(), $metric);
@@ -220,7 +223,7 @@ class UsageDB extends Usage
     /**
      * Collect Stats
      * Collect all database related stats
-     * 
+     *
      * @return void
      */
     public function collect(): void

From d39165889e1a633f6d2aa3ee7828b53d1cbf026e Mon Sep 17 00:00:00 2001
From: Damodar Lohani <dlohani48@gmail.com>
Date: Tue, 14 Jun 2022 00:40:33 +0000
Subject: [PATCH 4/7] error fixes

---
 src/Appwrite/Stats/Usage.php   | 4 ++--
 src/Appwrite/Stats/UsageDB.php | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php
index e5901ba8f9..ba1209faf9 100644
--- a/src/Appwrite/Stats/Usage.php
+++ b/src/Appwrite/Stats/Usage.php
@@ -186,7 +186,7 @@ class Usage
             if ($document->isEmpty()) {
                 $this->database->createDocument('stats', new Document([
                     '$id' => $id,
-                    'period' => $period['key'],
+                    'period' => $period,
                     'time' => $time,
                     'metric' => $metric,
                     'value' => $value,
@@ -199,7 +199,7 @@ class Usage
                     $document->setAttribute('value', $value)
                 );
             }
-            $this->latestTime[$metric][$period['key']] = $time;
+            $this->latestTime[$metric][$period] = $time;
         } catch (\Exception $e) { // if projects are deleted this might fail
             if (is_callable($this->errorHandler)) {
                 call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php
index 7165245497..b47956af82 100644
--- a/src/Appwrite/Stats/UsageDB.php
+++ b/src/Appwrite/Stats/UsageDB.php
@@ -34,7 +34,7 @@ class UsageDB extends Usage
                 if ($document->isEmpty()) {
                     $this->database->createDocument('stats', new Document([
                         '$id' => $id,
-                        'period' => $period['key'],
+                        'period' => $period,
                         'time' => $time,
                         'metric' => $metric,
                         'value' => $value,
@@ -179,7 +179,7 @@ class UsageDB extends Usage
         $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
             $metric = "storage.buckets.{$bucket->getId()}.files.count";
 
-            $count = $this->count($projectId, 'buckets_' . $bucket->getInternalId(), $metric);
+            $count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric);
             $projectFilesCount += $count;
 
             $metric = "storage.buckets.{$bucket->getId()}.files.total";

From ac13c8e79fcffc6f2caa6366bf6ad0f5f0dd0f96 Mon Sep 17 00:00:00 2001
From: Damodar Lohani <dlohani48@gmail.com>
Date: Tue, 14 Jun 2022 00:58:25 +0000
Subject: [PATCH 5/7] more error handling

---
 src/Appwrite/Stats/UsageDB.php | 40 +++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php
index b47956af82..1cdd34bb1e 100644
--- a/src/Appwrite/Stats/UsageDB.php
+++ b/src/Appwrite/Stats/UsageDB.php
@@ -77,7 +77,16 @@ class UsageDB extends Usage
         $this->database->setNamespace('_' . $projectId);
 
         while ($sum === $limit) {
-            $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument);
+            try {
+                $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument);
+            } catch (\Exception $e) {
+                if (is_callable($this->errorHandler)) {
+                    call_user_func($this->errorHandler, "Unable to fetch documents for project {$projectId} and collection {$collection}: {$e->getMessage()}", $e->getTraceAsString());
+                    return;
+                } else {
+                    throw $e;
+                }
+            }
             if (empty($results)) {
                 return;
             }
@@ -107,9 +116,17 @@ class UsageDB extends Usage
     private function sum(string $projectId, string $collection, string $attribute, string $metric): int
     {
         $this->database->setNamespace('_' . $projectId);
-        $sum = (int) $this->database->sum($collection, $attribute);
-        $this->createOrUpdateMetric($projectId, $metric, $sum);
-        return $sum;
+        try {
+            $sum = (int) $this->database->sum($collection, $attribute);
+            $this->createOrUpdateMetric($projectId, $metric, $sum);
+            return $sum;
+        } catch (\Exception $e) {
+            if (is_callable($this->errorHandler)) {
+                call_user_func($this->errorHandler, "Unable to fetch sum for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+            } else {
+                throw $e;
+            }
+        }
     }
 
     /**
@@ -125,10 +142,17 @@ class UsageDB extends Usage
     private function count(string $projectId, string $collection, string $metric): int
     {
         $this->database->setNamespace("_{$projectId}");
-        $count = $this->database->count($collection);
-
-        $this->createOrUpdateMetric($projectId, $metric, $count);
-        return $count;
+        try {
+            $count = $this->database->count($collection);
+            $this->createOrUpdateMetric($projectId, $metric, $count);
+            return $count;
+        } catch (\Exception $e) {
+            if (is_callable($this->errorHandler)) {
+                call_user_func($this->errorHandler, "Unable to fetch count for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+            } else {
+                throw $e;
+            }
+        }
     }
 
     /**

From ccb53ff1d005da45bd1dc9a7f7bf74a1b6831111 Mon Sep 17 00:00:00 2001
From: Damodar Lohani <lohanidamodar@users.noreply.github.com>
Date: Wed, 15 Jun 2022 05:08:23 +0545
Subject: [PATCH 6/7] Update src/Appwrite/Stats/Usage.php

Co-authored-by: Torsten Dittmann <torsten.dittmann@googlemail.com>
---
 src/Appwrite/Stats/Usage.php | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php
index ba1209faf9..c18f852cd6 100644
--- a/src/Appwrite/Stats/Usage.php
+++ b/src/Appwrite/Stats/Usage.php
@@ -237,7 +237,13 @@ class Usage
             $filters = '';
         }
 
-        $query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)";
+        $query = "SELECT sum(value) AS \"value\" ";
+        $query .= "FROM \"{$table}\" ";
+        $query .= "WHERE \"time\" > '{$start}' ";
+        $query .= "AND \"time\" < '{$end}' ";
+        $query .= "AND \"metric_type\"='counter' {$filters} ";
+        $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} ";
+        $query .= "FILL(null)";
         $result = $this->influxDB->query($query);
 
         $points = $result->getPoints();

From af19787ed9a331b1f8dcfed2c38a6e8dbc602080 Mon Sep 17 00:00:00 2001
From: Damodar Lohani <dlohani48@gmail.com>
Date: Tue, 14 Jun 2022 23:40:56 +0000
Subject: [PATCH 7/7] logging updates

---
 app/tasks/usage.php             |  36 +++++-
 app/views/install/compose.phtml |   2 +
 composer.lock                   | 206 ++++++++++++++++----------------
 docker-compose.yml              |   2 +
 src/Appwrite/Stats/Usage.php    |   4 +-
 src/Appwrite/Stats/UsageDB.php  |   8 +-
 6 files changed, 146 insertions(+), 112 deletions(-)

diff --git a/app/tasks/usage.php b/app/tasks/usage.php
index a46ab739b2..106ace9107 100644
--- a/app/tasks/usage.php
+++ b/app/tasks/usage.php
@@ -13,6 +13,7 @@ use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Database;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Registry\Registry;
+use Utopia\Logger\Log;
 
 function getDatabase(Registry &$register, string $namespace): Database
 {
@@ -72,9 +73,38 @@ function getInfluxDB(Registry &$register): InfluxDatabase
     return $database;
 }
 
-$logError = function ($message, $stackTrace) {
-    Console::warning("Failed: {$message}");
-    Console::warning($stackTrace);
+$logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) {
+    $logger = $register->get('logger');
+
+    if ($logger) {
+        $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
+
+        $log = new Log();
+        $log->setNamespace("realtime");
+        $log->setServer(\gethostname());
+        $log->setVersion($version);
+        $log->setType(Log::TYPE_ERROR);
+        $log->setMessage($error->getMessage());
+
+        $log->addTag('code', $error->getCode());
+        $log->addTag('verboseType', get_class($error));
+
+        $log->addExtra('file', $error->getFile());
+        $log->addExtra('line', $error->getLine());
+        $log->addExtra('trace', $error->getTraceAsString());
+        $log->addExtra('detailedTrace', $error->getTrace());
+
+        $log->setAction($action);
+
+        $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
+        $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
+
+        $responseCode = $logger->addLog($log);
+        Console::info('Usage stats log pushed with status code: ' . $responseCode);
+    }
+
+    Console::warning("Failed: {$error->getMessage()}");
+    Console::warning($error->getTraceAsString());
 };
 
 $cli
diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml
index 6fa8335bfa..cf9a111391 100644
--- a/app/views/install/compose.phtml
+++ b/app/views/install/compose.phtml
@@ -552,6 +552,8 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
 
   appwrite-schedule:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
diff --git a/composer.lock b/composer.lock
index 77a95ecc29..2f77c1c23b 100644
--- a/composer.lock
+++ b/composer.lock
@@ -1583,16 +1583,16 @@
         },
         {
             "name": "squizlabs/php_codesniffer",
-            "version": "3.6.2",
+            "version": "3.7.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
-                "reference": "5e4e71592f69da17871dba6e80dd51bce74a351a"
+                "reference": "a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/5e4e71592f69da17871dba6e80dd51bce74a351a",
-                "reference": "5e4e71592f69da17871dba6e80dd51bce74a351a",
+                "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563",
+                "reference": "a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563",
                 "shasum": ""
             },
             "require": {
@@ -1635,29 +1635,29 @@
                 "source": "https://github.com/squizlabs/PHP_CodeSniffer",
                 "wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki"
             },
-            "time": "2021-12-12T21:44:58+00:00"
+            "time": "2022-06-13T06:31:38+00:00"
         },
         {
             "name": "symfony/deprecation-contracts",
-            "version": "v2.5.1",
+            "version": "v3.1.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/deprecation-contracts.git",
-                "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66"
+                "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/e8b495ea28c1d97b5e0c121748d6f9b53d075c66",
-                "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66",
+                "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918",
+                "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918",
                 "shasum": ""
             },
             "require": {
-                "php": ">=7.1"
+                "php": ">=8.1"
             },
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-main": "2.5-dev"
+                    "dev-main": "3.1-dev"
                 },
                 "thanks": {
                     "name": "symfony/contracts",
@@ -1686,7 +1686,7 @@
             "description": "A generic function and convention to trigger deprecation notices",
             "homepage": "https://symfony.com",
             "support": {
-                "source": "https://github.com/symfony/deprecation-contracts/tree/v2.5.1"
+                "source": "https://github.com/symfony/deprecation-contracts/tree/v3.1.0"
             },
             "funding": [
                 {
@@ -1702,89 +1702,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2022-01-02T09:53:40+00:00"
-        },
-        {
-            "name": "symfony/polyfill-ctype",
-            "version": "v1.26.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/symfony/polyfill-ctype.git",
-                "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4",
-                "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=7.1"
-            },
-            "provide": {
-                "ext-ctype": "*"
-            },
-            "suggest": {
-                "ext-ctype": "For best performance"
-            },
-            "type": "library",
-            "extra": {
-                "branch-alias": {
-                    "dev-main": "1.26-dev"
-                },
-                "thanks": {
-                    "name": "symfony/polyfill",
-                    "url": "https://github.com/symfony/polyfill"
-                }
-            },
-            "autoload": {
-                "files": [
-                    "bootstrap.php"
-                ],
-                "psr-4": {
-                    "Symfony\\Polyfill\\Ctype\\": ""
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Gert de Pagter",
-                    "email": "BackEndTea@gmail.com"
-                },
-                {
-                    "name": "Symfony Community",
-                    "homepage": "https://symfony.com/contributors"
-                }
-            ],
-            "description": "Symfony polyfill for ctype functions",
-            "homepage": "https://symfony.com",
-            "keywords": [
-                "compatibility",
-                "ctype",
-                "polyfill",
-                "portable"
-            ],
-            "support": {
-                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0"
-            },
-            "funding": [
-                {
-                    "url": "https://symfony.com/sponsor",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/fabpot",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2022-05-24T11:49:31+00:00"
+            "time": "2022-02-25T11:15:52+00:00"
         },
         {
             "name": "symfony/polyfill-php80",
@@ -2905,21 +2823,21 @@
         },
         {
             "name": "webmozart/assert",
-            "version": "1.10.0",
+            "version": "1.11.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/webmozarts/assert.git",
-                "reference": "6964c76c7804814a842473e0c8fd15bab0f18e25"
+                "reference": "11cb2199493b2f8a3b53e7f19068fc6aac760991"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/webmozarts/assert/zipball/6964c76c7804814a842473e0c8fd15bab0f18e25",
-                "reference": "6964c76c7804814a842473e0c8fd15bab0f18e25",
+                "url": "https://api.github.com/repos/webmozarts/assert/zipball/11cb2199493b2f8a3b53e7f19068fc6aac760991",
+                "reference": "11cb2199493b2f8a3b53e7f19068fc6aac760991",
                 "shasum": ""
             },
             "require": {
-                "php": "^7.2 || ^8.0",
-                "symfony/polyfill-ctype": "^1.8"
+                "ext-ctype": "*",
+                "php": "^7.2 || ^8.0"
             },
             "conflict": {
                 "phpstan/phpstan": "<0.12.20",
@@ -2957,9 +2875,9 @@
             ],
             "support": {
                 "issues": "https://github.com/webmozarts/assert/issues",
-                "source": "https://github.com/webmozarts/assert/tree/1.10.0"
+                "source": "https://github.com/webmozarts/assert/tree/1.11.0"
             },
-            "time": "2021-03-09T10:59:23+00:00"
+            "time": "2022-06-03T18:03:27+00:00"
         }
     ],
     "packages-dev": [
@@ -5086,6 +5004,88 @@
             ],
             "time": "2022-04-18T20:38:04+00:00"
         },
+        {
+            "name": "symfony/polyfill-ctype",
+            "version": "v1.26.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/polyfill-ctype.git",
+                "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4",
+                "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=7.1"
+            },
+            "provide": {
+                "ext-ctype": "*"
+            },
+            "suggest": {
+                "ext-ctype": "For best performance"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-main": "1.26-dev"
+                },
+                "thanks": {
+                    "name": "symfony/polyfill",
+                    "url": "https://github.com/symfony/polyfill"
+                }
+            },
+            "autoload": {
+                "files": [
+                    "bootstrap.php"
+                ],
+                "psr-4": {
+                    "Symfony\\Polyfill\\Ctype\\": ""
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Gert de Pagter",
+                    "email": "BackEndTea@gmail.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Symfony polyfill for ctype functions",
+            "homepage": "https://symfony.com",
+            "keywords": [
+                "compatibility",
+                "ctype",
+                "polyfill",
+                "portable"
+            ],
+            "support": {
+                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2022-05-24T11:49:31+00:00"
+        },
         {
             "name": "symfony/polyfill-mbstring",
             "version": "v1.26.0",
diff --git a/docker-compose.yml b/docker-compose.yml
index b6e1ed68b2..c5e11f1c51 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -588,6 +588,8 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
 
   appwrite-schedule:
     entrypoint: schedule
diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php
index c18f852cd6..9a885c011f 100644
--- a/src/Appwrite/Stats/Usage.php
+++ b/src/Appwrite/Stats/Usage.php
@@ -202,7 +202,7 @@ class Usage
             $this->latestTime[$metric][$period] = $time;
         } catch (\Exception $e) { // if projects are deleted this might fail
             if (is_callable($this->errorHandler)) {
-                call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+                call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
             } else {
                 throw $e;
             }
@@ -290,7 +290,7 @@ class Usage
                     $this->syncFromInfluxDB($metric, $options, $period);
                 } catch (\Exception $e) {
                     if (is_callable($this->errorHandler)) {
-                        call_user_func($this->errorHandler, $e->getMessage(), $e->getTraceAsString());
+                        call_user_func($this->errorHandler, $e);
                     } else {
                         throw $e;
                     }
diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php
index 1cdd34bb1e..cc312b69ac 100644
--- a/src/Appwrite/Stats/UsageDB.php
+++ b/src/Appwrite/Stats/UsageDB.php
@@ -49,7 +49,7 @@ class UsageDB extends Usage
                 }
             } catch (\Exception$e) { // if projects are deleted this might fail
                 if (is_callable($this->errorHandler)) {
-                    call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+                    call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
                 } else {
                     throw $e;
                 }
@@ -81,7 +81,7 @@ class UsageDB extends Usage
                 $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument);
             } catch (\Exception $e) {
                 if (is_callable($this->errorHandler)) {
-                    call_user_func($this->errorHandler, "Unable to fetch documents for project {$projectId} and collection {$collection}: {$e->getMessage()}", $e->getTraceAsString());
+                    call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}");
                     return;
                 } else {
                     throw $e;
@@ -122,7 +122,7 @@ class UsageDB extends Usage
             return $sum;
         } catch (\Exception $e) {
             if (is_callable($this->errorHandler)) {
-                call_user_func($this->errorHandler, "Unable to fetch sum for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+                call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}");
             } else {
                 throw $e;
             }
@@ -148,7 +148,7 @@ class UsageDB extends Usage
             return $count;
         } catch (\Exception $e) {
             if (is_callable($this->errorHandler)) {
-                call_user_func($this->errorHandler, "Unable to fetch count for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString());
+                call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}");
             } else {
                 throw $e;
             }