Merge remote-tracking branch 'origin/feat-database-indexing' into fix-remove-local-swagger-validator

app/http.php

@@ -54,8 +54,25 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
     $app = new App('UTC');
 
     go(function() use ($register, $app) {
-        $db = $register->get('dbPool')->get();
-        $redis = $register->get('redisPool')->get();
+        // wait for database to be ready
+        $attempts = 0;
+        $max = 10;
+        $sleep = 1;
+
+        do {
+            try {
+                $attempts++;
+                $db = $register->get('dbPool')->get();
+                $redis = $register->get('redisPool')->get();
+                break; // leave the do-while if successful
+            } catch(\Exception $e) {
+                Console::warning("Database not ready. Retrying connection ({$attempts})...");
+                if ($attempts >= $max) {
+                    throw new \Exception('Failed to connect to database: '. $e->getMessage());
+                }
+                sleep($sleep);
+            }
+        } while ($attempts < $max);
 
         App::setResource('db', function () use (&$db) {
             return $db;
@@ -69,9 +86,6 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
             return $app;
         });
 
-        // wait for database to be ready
-        sleep(5);
-
         $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */
 
         if(!$dbForConsole->exists()) {

app/workers.php

@@ -23,10 +23,12 @@ $register->set('db', function () {
 
     return $pdo;
 });
+
 $register->set('cache', function () { // Register cache connection
     $redis = new Redis();
     $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
     $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
 
     return $redis;
-});
+});
+

app/workers/audits.php

@@ -2,11 +2,7 @@
 
 use Appwrite\Resque\Worker;
 use Utopia\Audit\Audit;
-use Utopia\Cache\Adapter\Redis;
-use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
-use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Database;
 
 require_once __DIR__.'/../workers.php';
 
@@ -23,8 +19,6 @@ class AuditsV1 extends Worker
 
     public function run(): void
     {
-        global $register;
-
         $projectId = $this->args['projectId'];
         $userId = $this->args['userId'];
         $event = $this->args['event'];
@@ -32,12 +26,8 @@ class AuditsV1 extends Worker
         $userAgent = $this->args['userAgent'];
         $ip = $this->args['ip'];
         $data = $this->args['data'];
-        $db = $register->get('db', true);
         
-        $cache = new Cache(new Redis($register->get('cache')));
-        $dbForInternal = new Database(new MariaDB($db), $cache);
-        $dbForInternal->setNamespace('project_'.$projectId.'_internal');
-
+        $dbForInternal = $this->getInternalDB($projectId);
         $audit = new Audit($dbForInternal);
 
         $audit->log($userId, $event, $resource, $userAgent, $ip, '', $data);

app/workers/database.php

@@ -1,14 +1,10 @@
 <?php
 
 use Appwrite\Resque\Worker;
-use Utopia\Cache\Cache;
-use Utopia\Cache\Adapter\Redis as RedisCache;
 use Utopia\CLI\Console;
-use Utopia\Database\Database;
 use Utopia\Database\Document;
-use Utopia\Database\Adapter\MariaDB;
 
-require_once __DIR__.'/../init.php';
+require_once __DIR__.'/../workers.php';
 
 Console::title('Database V1 Worker');
 Console::success(APP_NAME.' database worker v1 has started'."\n");
@@ -131,53 +127,4 @@ class DatabaseV1 extends Worker
 
         $success = $dbForExternal->deleteIndex($collectionId, $id);
     }
-
-    /**
-     * @param string $projectId
-     *
-     * @return Database
-     */
-    protected function getInternalDB($projectId): Database
-    {
-        global $register;
-
-        $dbForInternal = null;
-
-        go(function() use ($register, $projectId, &$dbForInternal) {
-            $db = $register->get('dbPool')->get();
-            $redis = $register->get('redisPool')->get();
-
-            $cache = new Cache(new RedisCache($redis));
-            $dbForInternal = new Database(new MariaDB($db), $cache);
-            $dbForInternal->setNamespace('project_'.$projectId.'_internal'); // Main DB
-
-        });
-
-        return $dbForInternal;
-    }
-
-    /**
-     * @param string $projectId
-     *
-     * @return Database
-     */
-    protected function getExternalDB($projectId): Database
-    {
-        global $register;
-
-        /** @var Database $dbForExternal */
-        $dbForExternal = null;
-
-        go(function() use ($register, $projectId, &$dbForExternal) {
-            $db = $register->get('dbPool')->get();
-            $redis = $register->get('redisPool')->get();
-
-            $cache = new Cache(new RedisCache($redis));
-            $dbForExternal = new Database(new MariaDB($db), $cache);
-            $dbForExternal->setNamespace('project_'.$projectId.'_external'); // Main DB
-
-        });
-
-        return $dbForExternal;
-    }
 }

app/workers/deletes.php

@@ -3,17 +3,13 @@
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
-use Utopia\Cache\Adapter\Redis as RedisCache;
 use Utopia\Database\Validator\Authorization;
 use Appwrite\Resque\Worker;
 use Utopia\Storage\Device\Local;
 use Utopia\Abuse\Abuse;
 use Utopia\Abuse\Adapters\TimeLimit;
 use Utopia\CLI\Console;
-use Utopia\Config\Config;
 use Utopia\Audit\Audit;
-use Utopia\Cache\Cache;
-use Utopia\Database\Adapter\MariaDB;
 
 require_once __DIR__.'/../workers.php';
 
@@ -351,48 +347,4 @@ class DeletesV1 extends Worker
             Console::info("No certificate files found for {$domain}");
         }
     }
-    
-    /**
-     * @param string $projectId
-     * @return Database
-     */
-    protected function getInternalDB($projectId): Database
-    {
-        global $register;
-        
-        $cache = new Cache(new RedisCache($register->get('cache')));
-        $dbForInternal = new Database(new MariaDB($register->get('db')), $cache);
-        $dbForInternal->setNamespace('project_'.$projectId.'_internal'); // Main DB
-
-        return $dbForInternal;
-    }
-
-    /**
-     * @param string $projectId
-     * @return Database
-     */
-    protected function getExternalDB($projectId): Database
-    {
-        global $register;
-
-        $cache = new Cache(new RedisCache($register->get('cache')));
-        $dbForExternal = new Database(new MariaDB($register->get('db')), $cache);
-        $dbForExternal->setNamespace('project_'.$projectId.'_external'); // Main DB
-
-        return $dbForExternal;
-    }
-
-    /**
-     * @return Database
-     */
-    protected function getConsoleDB(): Database
-    {
-        global $register;
-
-        $cache = new Cache(new RedisCache($register->get('cache')));
-        $dbForConsole = new Database(new MariaDB($register->get('db')), $cache);
-        $dbForConsole->setNamespace('project_console_internal'); // Main DB
-
-        return $dbForConsole;
-    }
 }

app/workers/functions.php

@@ -6,11 +6,8 @@ use Appwrite\Utopia\Response\Model\Execution;
 use Cron\CronExpression;
 use Swoole\Runtime;
 use Utopia\App;
-use Utopia\Cache\Adapter\Redis;
-use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
-use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
@@ -139,9 +136,6 @@ class FunctionsV1 extends Worker
     {
         global $register;
 
-        $db = $register->get('db');
-        $cache = $register->get('cache');
-
         $projectId = $this->args['projectId'] ?? '';
         $functionId = $this->args['functionId'] ?? '';
         $webhooks = $this->args['webhooks'] ?? [];
@@ -154,9 +148,7 @@ class FunctionsV1 extends Worker
         $userId = $this->args['userId'] ?? '';
         $jwt = $this->args['jwt'] ?? '';
 
-        $cache = new Cache(new Redis($cache));
-        $database = new Database(new MariaDB($db), $cache);
-        $database->setNamespace('project_'.$projectId.'_internal');
+        $database = $this->getInternalDB($projectId);
 
         switch ($trigger) {
             case 'event':

app/workers/tasks.php

@@ -28,11 +28,6 @@ class TasksV1 extends Worker
 
     public function run(): void
     {
-        global $register;
-
-        $db = $register->get('db');
-        $cache = $register->get('cache');
-
         $projectId = $this->args['projectId'] ?? null;
         $taskId = $this->args['$id'] ?? null;
         $updated = $this->args['updated'] ?? null;
@@ -44,9 +39,7 @@ class TasksV1 extends Worker
         $logLimit = 5;
         $alert = '';
 
-        $cache = new Cache(new Redis($cache));
-        $dbForConsole = new Database(new MariaDB($db), $cache);
-        $dbForConsole->setNamespace('project_console_internal');
+        $dbForConsole = $this->getConsoleDB();
 
         /*
          * 1. Get Original Task

src/Appwrite/Resque/Worker.php

@@ -2,6 +2,12 @@
 
 namespace Appwrite\Resque;
 
+use Utopia\Cache\Cache;
+use Utopia\Cache\Adapter\Redis as RedisCache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Adapter\MariaDB;
+
 abstract class Worker
 {
     public $args = [];
@@ -12,6 +18,13 @@ abstract class Worker
 
     abstract public function shutdown(): void;
 
+    const MAX_ATTEMPTS = 10;
+    const SLEEP_TIME = 2;
+
+    const DATABASE_INTERNAL = 'internal';
+    const DATABASE_EXTERNAL = 'external';
+    const DATABASE_CONSOLE = 'console';
+
     public function setUp(): void
     {
         $this->init();
@@ -26,4 +39,91 @@ abstract class Worker
     {
         $this->shutdown();
     }
+    /**
+     * Get internal project database
+     * @param string $projectId
+     * @return Database
+     */
+    protected function getInternalDB(string $projectId): Database
+    {
+        return $this->getDB(self::DATABASE_INTERNAL, $projectId);
+    }
+
+    /**
+     * Get external project database
+     * @param string $projectId
+     * @return Database
+     */
+    protected function getExternalDB(string $projectId): Database
+    {
+        return $this->getDB(self::DATABASE_EXTERNAL, $projectId);
+    }
+
+    /**
+     * Get console database
+     * @return Database
+     */
+    protected function getConsoleDB(): Database
+    {
+        return $this->getDB(self::DATABASE_CONSOLE);
+    }
+
+    /**
+     * Get console database
+     * @param string $type One of (internal, external, console)
+     * @param string $projectId of internal or external DB
+     * @return Database
+     */
+    private function getDB($type, $projectId = ''): Database
+    {
+        global $register;
+
+        $namespace = '';
+        $sleep = self::SLEEP_TIME; // overwritten when necessary
+
+        switch ($type) {
+            case self::DATABASE_INTERNAL:
+                if (!$projectId) {
+                    throw new \Exception('ProjectID not provided - cannot get database');
+                }
+                $namespace = "project_{$projectId}_internal";
+                break;
+            case self::DATABASE_EXTERNAL:
+                if (!$projectId) {
+                    throw new \Exception('ProjectID not provided - cannot get database');
+                }
+                $namespace = "project_{$projectId}_external";
+                break;
+            case self::DATABASE_CONSOLE:
+                $namespace = "project_console_internal";
+                $sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created
+                break;
+            default:
+                throw new \Exception('Unknown database type: ' . $type);
+                break;
+        }
+
+        $attempts = 0;
+
+        do {
+            try {
+                $attempts++;
+                $cache = new Cache(new RedisCache($register->get('cache')));
+                $database = new Database(new MariaDB($register->get('db')), $cache);
+                $database->setNamespace($namespace); // Main DB
+                if (!$database->exists()) {
+                    throw new \Exception("Table does not exist: {$database->getNamespace()}");
+                }
+                break; // leave loop if successful
+            } catch(\Exception $e) {
+                Console::warning("Database not ready. Retrying connection ({$attempts})...");
+                if ($attempts >= self::MAX_ATTEMPTS) {
+                    throw new \Exception('Failed to connect to database: '. $e->getMessage());
+                }
+                sleep($sleep);
+            }
+        } while ($attempts < self::MAX_ATTEMPTS);
+
+        return $database;
+    }
 }