From 1cf7da26d4aff9b22b7f0729b5359cc227bb59c1 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 19 Jan 2022 17:15:22 -0500 Subject: [PATCH 1/4] Test new exists behavior in database --- composer.json | 2 +- composer.lock | 44 +++++++++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/composer.json b/composer.json index d6c0a7ff2d..016c33888b 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.13.*", + "utopia-php/database": "dev-feat-check-if-collection-exists as 0.14.0", "utopia-php/locale": "0.4.*", "utopia-php/orchestration": "0.2.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 62a9f81bd6..644b6bc749 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f022f43cc2d6023c3dad3805b7c4455a", + "content-hash": "f864e323b153e24e1392840eb114b951", "packages": [ { "name": "adhocore/jwt", @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.4", + "version": "1.11.99.5", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "b174585d1fe49ceed21928a945138948cb394600" + "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b174585d1fe49ceed21928a945138948cb394600", - "reference": "b174585d1fe49ceed21928a945138948cb394600", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b4f54f74ef3453349c24a845d22392cd31e65f1d", + "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.4" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.5" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-09-13T08:41:34+00:00" + "time": "2022-01-17T14:14:24+00:00" }, { "name": "dragonmantank/cron-expression", @@ -2141,16 +2141,16 @@ }, { "name": "utopia-php/database", - "version": "0.13.2", + "version": "dev-feat-check-if-collection-exists", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "bf92279b707b3a10ee5ec5df5c065023b2221357" + "reference": "49b4eef6728f44a33df2707993319a2f352e75fa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/bf92279b707b3a10ee5ec5df5c065023b2221357", - "reference": "bf92279b707b3a10ee5ec5df5c065023b2221357", + "url": "https://api.github.com/repos/utopia-php/database/zipball/49b4eef6728f44a33df2707993319a2f352e75fa", + "reference": "49b4eef6728f44a33df2707993319a2f352e75fa", "shasum": "" }, "require": { @@ -2198,9 +2198,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.13.2" + "source": "https://github.com/utopia-php/database/tree/feat-check-if-collection-exists" }, - "time": "2022-01-04T10:51:22+00:00" + "time": "2022-01-19T21:01:38+00:00" }, { "name": "utopia-php/domains", @@ -3703,9 +3703,6 @@ "require": { "php": "^7.1 || ^8.0" }, - "replace": { - "myclabs/deep-copy": "self.version" - }, "require-dev": { "doctrine/collections": "^1.0", "doctrine/common": "^2.6", @@ -6645,9 +6642,18 @@ "time": "2015-12-17T08:42:14+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-check-if-collection-exists", + "alias": "0.14.0", + "alias_normalized": "0.14.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -6669,5 +6675,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.2.0" } From 4a8c87970661124de6f62ff54e2be4ac1c6209ae Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 19 Jan 2022 17:15:38 -0500 Subject: [PATCH 2/4] Retry database connections in realtime server --- app/realtime.php | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 9acb37570d..e783f166de 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -91,13 +91,34 @@ $server->error($logError); function getDatabase(Registry &$register, string $namespace) { - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); + $attempts = 0; + $sleep = 2; + $maxAttempts = 10; - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); + do { + try { + $attempts++; + + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + + $cache = new Cache(new RedisCache($redis)); + $database = new Database(new MariaDB($db), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace($namespace); + + if (!$database->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'realtime')) { + throw new Exception('Collection not ready'); + } + break; // leave loop if successful + } catch(\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $maxAttempts) { + throw new \Exception('Failed to connect to database: '. $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $maxAttempts); return [ $database, @@ -106,6 +127,7 @@ function getDatabase(Registry &$register, string $namespace) $register->get('redisPool')->put($redis); } ]; + }; $server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) { From f7373fff2b10d8cfde1bda2e62998f71dbe2e362 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 21 Jan 2022 11:39:21 -0500 Subject: [PATCH 3/4] Update utopia/database to latest --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index 016c33888b..e7bd6646a9 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-check-if-collection-exists as 0.14.0", + "utopia-php/database": "0.14.*", "utopia-php/locale": "0.4.*", "utopia-php/orchestration": "0.2.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 644b6bc749..91440e4a7a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f864e323b153e24e1392840eb114b951", + "content-hash": "ab493f0a7f01a1105f8bc5caaf9b928b", "packages": [ { "name": "adhocore/jwt", @@ -2141,16 +2141,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-check-if-collection-exists", + "version": "0.14.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "49b4eef6728f44a33df2707993319a2f352e75fa" + "reference": "2f2527bb080cf578fba327ea2ec637064561d403" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/49b4eef6728f44a33df2707993319a2f352e75fa", - "reference": "49b4eef6728f44a33df2707993319a2f352e75fa", + "url": "https://api.github.com/repos/utopia-php/database/zipball/2f2527bb080cf578fba327ea2ec637064561d403", + "reference": "2f2527bb080cf578fba327ea2ec637064561d403", "shasum": "" }, "require": { @@ -2198,9 +2198,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-check-if-collection-exists" + "source": "https://github.com/utopia-php/database/tree/0.14.0" }, - "time": "2022-01-19T21:01:38+00:00" + "time": "2022-01-21T16:34:34+00:00" }, { "name": "utopia-php/domains", @@ -6642,18 +6642,9 @@ "time": "2015-12-17T08:42:14+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-check-if-collection-exists", - "alias": "0.14.0", - "alias_normalized": "0.14.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 7e6c9b9d87e0348b158cc24a21034d8caf808b5a Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 21 Jan 2022 12:32:27 -0500 Subject: [PATCH 4/4] Harmonize database reconnect across services --- app/init.php | 3 +++ app/realtime.php | 10 ++++------ src/Appwrite/Resque/Worker.php | 15 +++++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/app/init.php b/app/init.php index 439e7cc3b1..d591d1ddbd 100644 --- a/app/init.php +++ b/app/init.php @@ -86,6 +86,9 @@ const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244'; const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; +// Database Reconnect +const DATABASE_RECONNECT_SLEEP = 2; +const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; // Database Worker Types const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; diff --git a/app/realtime.php b/app/realtime.php index e783f166de..b2ddf67634 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -92,8 +92,6 @@ $server->error($logError); function getDatabase(Registry &$register, string $namespace) { $attempts = 0; - $sleep = 2; - $maxAttempts = 10; do { try { @@ -107,18 +105,18 @@ function getDatabase(Registry &$register, string $namespace) $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $database->setNamespace($namespace); - if (!$database->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'realtime')) { + if (!$database->exists($database->getDefaultDatabase(), 'realtime')) { throw new Exception('Collection not ready'); } break; // leave loop if successful } catch(\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $maxAttempts) { + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { throw new \Exception('Failed to connect to database: '. $e->getMessage()); } - sleep($sleep); + sleep(DATABASE_RECONNECT_SLEEP); } - } while ($attempts < $maxAttempts); + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); return [ $database, diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 1444c90663..b9ed5af275 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -70,9 +70,6 @@ abstract class Worker throw new Exception("Please implement getName method in worker"); } - const MAX_ATTEMPTS = 10; - const SLEEP_TIME = 2; - const DATABASE_PROJECT = 'project'; const DATABASE_CONSOLE = 'console'; @@ -174,7 +171,7 @@ abstract class Worker global $register; $namespace = ''; - $sleep = self::SLEEP_TIME; // overwritten when necessary + $sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary switch ($type) { case self::DATABASE_PROJECT: @@ -201,18 +198,24 @@ abstract class Worker $database = new Database(new MariaDB($register->get('db')), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $database->setNamespace($namespace); // Main DB + if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) { throw new \Exception("Project does not exist: {$projectId}"); } + + if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), 'realtime')) { + throw new \Exception('Console project not ready'); + } + break; // leave loop if successful } catch(\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= self::MAX_ATTEMPTS) { + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { throw new \Exception('Failed to connect to database: '. $e->getMessage()); } sleep($sleep); } - } while ($attempts < self::MAX_ATTEMPTS); + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); return $database; }