Merge pull request #5084 from appwrite/automate-hamster-script

feat: automate hamster

.env

@@ -46,6 +46,7 @@ _APP_SMTP_SECURE=
 _APP_SMTP_USERNAME=
 _APP_SMTP_PASSWORD=
 _APP_HAMSTER_RECIPIENTS=
+_APP_HAMSTER_INTERVAL=86400
 _APP_SMS_PROVIDER=sms://username:password@mock
 _APP_SMS_FROM=+123456789
 _APP_STORAGE_LIMIT=30000000

app/console

@@ -1 +1 @@
-Subproject commit aea8c5f2bbe0836fc9fb4720b21d44fe01ac93d1
+Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1

docker-compose.yml

@@ -544,6 +544,39 @@ services:
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
+  appwrite-hamster:
+    entrypoint: hamster
+    <<: *x-logging
+    container_name: appwrite-hamster
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_CONNECTIONS_MAX
+      - _APP_POOL_CLIENTS
+      - _APP_OPENSSL_KEY_V1
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_HAMSTER_RECIPIENTS
+      - _APP_HAMSTER_INTERVAL
+
   appwrite-maintenance:
     entrypoint: maintenance
     <<: *x-logging

src/Appwrite/Platform/Tasks/Hamster.php

@@ -145,79 +145,94 @@ class Hamster extends Action
 
     public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void
     {
-        Console::info('Getting stats...');
 
-        /* Initialise new Utopia app */
+        Console::title('Cloud Hamster V1');
-        $app = new App('UTC');
+        Console::success(APP_NAME . ' cloud hamster process v1 has started');
-        $console = $app->getResource('console');
 
-        /** CSV stuff */
+        $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
-        $this->date = date('Y-m-d');
-        $this->path = "{$this->directory}/stats_{$this->date}.csv";
-        $csv = Writer::createFromPath($this->path, 'w');
-        $csv->insertOne($this->columns);
 
-        /** Database connections */
+        Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $interval) {
-        $totalProjects = $dbForConsole->count('projects') + 1;
-        Console::success("Found a total of: {$totalProjects} projects");
 
-        $projects = [$console];
+            $now = date('d-m-Y H:i:s', time());
-        $count = 0;
+            Console::info("[{$now}] Getting Cloud Usage Stats every {$interval} seconds");
-        $limit = 30;
+            $loopStart = microtime(true);
-        $sum = 30;
+
-        $offset = 0;
+            /* Initialise new Utopia app */
-        while (!empty($projects)) {
+            $app = new App('UTC');
-            foreach ($projects as $project) {
+            $console = $app->getResource('console');
-                /**
+
-                 * Skip user projects with id 'console'
+            /** CSV stuff */
-                 */
+            $this->date = date('Y-m-d');
-                if ($project->getId() === 'console') {
+            $this->path = "{$this->directory}/stats_{$this->date}.csv";
-                    continue;
+            $csv = Writer::createFromPath($this->path, 'w');
+            $csv->insertOne($this->columns);
+
+            /** Database connections */
+            $totalProjects = $dbForConsole->count('projects') + 1;
+            Console::success("Found a total of: {$totalProjects} projects");
+
+            $projects = [$console];
+            $count = 0;
+            $limit = 30;
+            $sum = 30;
+            $offset = 0;
+            while (!empty($projects)) {
+                foreach ($projects as $project) {
+                    /**
+                     * Skip user projects with id 'console'
+                     */
+                    if ($project->getId() === 'console') {
+                        continue;
+                    }
+
+                    Console::info("Getting stats for {$project->getId()}");
+
+                    try {
+                        $db = $project->getAttribute('database');
+                        $adapter = $pools
+                            ->get($db)
+                            ->pop()
+                            ->getResource();
+
+                        $dbForProject = new Database($adapter, $cache);
+                        $dbForProject->setDefaultDatabase('appwrite');
+                        $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                        $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project);
+                        $csv->insertOne(array_values($statsPerProject));
+                    } catch (\Throwable $th) {
+                        throw $th;
+                        Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
+                    } finally {
+                        $pools
+                            ->get($db)
+                            ->reclaim();
+                    }
                 }
 
-                Console::info("Getting stats for {$project->getId()}");
+                $sum = \count($projects);
 
-                try {
+                $projects = $dbForConsole->find('projects', [
-                    $db = $project->getAttribute('database');
+                    Query::limit($limit),
-                    $adapter = $pools
+                    Query::offset($offset),
-                        ->get($db)
+                ]);
-                        ->pop()
-                        ->getResource();
 
-                    $dbForProject = new Database($adapter, $cache);
+                $offset = $offset + $limit;
-                    $dbForProject->setDefaultDatabase('appwrite');
+                $count = $count + $sum;
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
 
-                    $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project);
+                Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
-                    $csv->insertOne(array_values($statsPerProject));
-                } catch (\Throwable $th) {
-                    throw $th;
-                    Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
             }
 
-            $sum = \count($projects);
+            $this->sendEmail($register);
 
-            $projects = $dbForConsole->find('projects', [
+            $pools
-                Query::limit($limit),
+                ->get('console')
-                Query::offset($offset),
+                ->reclaim();
-            ]);
 
-            $offset = $offset + $limit;
+            $loopTook = microtime(true) - $loopStart;
-            $count = $count + $sum;
+            $now = date('d-m-Y H:i:s', time());
-
+            Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
-            Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
+        }, $interval);
-        }
-
-        $this->sendEmail($register);
-
-        $pools
-            ->get('console')
-            ->reclaim();
     }
 
     private function sendEmail(Registry $register)