Merge pull request #4673 from appwrite/feat-realtime-dbpools

Feat realtime dbpools

.env

@@ -23,6 +23,7 @@ _APP_DB_SCHEMA=appwrite
 _APP_DB_USER=user
 _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
+_APP_DB_MAX_CONNECTIONS=251
 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
 _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
 _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379

CHANGES.md

@@ -1,6 +1,9 @@
-# Version 1.2.0
+# TBD
 
 - Replace Appwrite executor with OpenRuntimes Executor [#4650](https://github.com/appwrite/appwrite/pull/4650)
+- Add `_APP_DB_MAX_CONNECTIONS` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
+- Update Traefik 2.7 -> 2.9 [#4673](https://github.com/appwrite/appwrite/pull/4673)
+- Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673)
 
 # Version 1.1.0
 

app/config/variables.php

@@ -306,24 +306,33 @@ return [
                 'question' => '',
                 'filter' => 'password'
             ],
-            // [
-            //     'name' => '_APP_CONNECTIONS_DB_PROJECT',
-            //     'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
-            //     'introduction' => 'TBD',
-            //     'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
-            //     'required' => true,
-            //     'question' => '',
-            //     'filter' => ''
-            // ],
-            // [
-            //     'name' => '_APP_CONNECTIONS_DB_CONSOLE',
-            //     'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
-            //     'introduction' => 'TBD',
-            //     'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
-            //     'required' => true,
-            //     'question' => '',
-            //     'filter' => ''
-            // ]
+            [
+                'name' => '_APP_DB_MAX_CONNECTIONS',
+                'description' => 'MariaDB server maximum connections.',
+                'introduction' => 'TBD',
+                'default' => 251,
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+             [
+                 'name' => '_APP_CONNECTIONS_DB_PROJECT',
+                 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
+                 'introduction' => 'TBD',
+                 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
+                 'required' => true,
+                 'question' => '',
+                 'filter' => ''
+             ],
+             [
+                 'name' => '_APP_CONNECTIONS_DB_CONSOLE',
+                 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
+                 'introduction' => 'TBD',
+                 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
+                 'required' => true,
+                 'question' => '',
+                 'filter' => ''
+             ]
         ],
     ],
     [

app/init.php

@@ -97,6 +97,7 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
 const APP_CACHE_BUSTER = 501;
 const APP_VERSION_STABLE = '1.0.3';
+const APP_DEFAULT_POOL_SIZE = 64;
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
 const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -496,7 +497,7 @@ $register->set('logger', function () {
     $adapter = new $classname($providerConfig);
     return new Logger($adapter);
 });
-$register->set('pools', function () {
+$register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
     $group = new Group();
 
     $fallbackForDB = AppwriteURL::unparse([
@@ -620,7 +621,7 @@ $register->set('pools', function () {
                     break;
             }
 
-            $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) {
+            $pool = new Pool($name, $size, function () use ($type, $resource, $dsn) {
                 // Get Adapter
                 $adapter = null;
 
@@ -1122,6 +1123,26 @@ function getDevice($root): Device
     }
 }
 
+/**
+ * Get database connection pool size for worker processes.
+ *
+ * @return int
+ * @throws \Exception
+ */
+function getWorkerPoolSize(): int
+{
+    $reservedConnections = APP_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API
+    $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
+    $maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 251);
+    $workerConnections = $maxConnections - $reservedConnections;
+
+    if ($workerCount > $workerConnections) {
+        throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
+    }
+
+    return (int)($workerConnections / $workerCount);
+}
+
 App::setResource('mode', function ($request) {
     /** @var Appwrite\Utopia\Request $request */
 

app/realtime.php

@@ -36,7 +36,8 @@ function getConsoleDB(): Database
 {
     global $register;
 
-    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+    /** @var \Utopia\Pools\Group $pools */
+    $pools = $register->get('pools', args: [getWorkerPoolSize()]);
 
     $dbAdapter = $pools
         ->get('console')
@@ -55,7 +56,8 @@ function getProjectDB(Document $project): Database
 {
     global $register;
 
-    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+    /** @var \Utopia\Pools\Group $pools */
+    $pools = $register->get('pools', args: [getWorkerPoolSize()]);
 
     if ($project->isEmpty() || $project->getId() === 'console') {
         return getConsoleDB();
@@ -364,10 +366,11 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
             call_user_func($logError, $th, "pubSubConnection");
 
             Console::error('Pub/sub error: ' . $th->getMessage());
-            $register->get('pools')->reclaim();
             $attempts++;
             sleep(DATABASE_RECONNECT_SLEEP);
             continue;
+        } finally {
+            $register->get('pools')->reclaim();
         }
     }
 
@@ -556,7 +559,6 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
 
             default:
                 throw new Exception('Message type is not valid.', 1003);
-                break;
         }
     } catch (\Throwable $th) {
         $response = [

app/views/install/compose.phtml

@@ -15,7 +15,7 @@ $image = $this->getParam('image', '');
 
 services:
   traefik:
-    image: traefik:2.7
+    image: traefik:2.9
     container_name: appwrite-traefik
     <<: *x-logging
     command:
@@ -30,6 +30,15 @@ services:
     ports:
       - <?php echo $httpPort; ?>:80
       - <?php echo $httpsPort; ?>:443
+    ulimits:
+      nofile:
+        soft: 655350
+        hard: 655350
+    sysctls:
+      - net.core.somaxconn=1024
+      - net.ipv4.tcp_rmem=1024 4096 16384
+      - net.ipv4.tcp_wmem=1024 4096 16384
+      - net.ipv4.ip_local_port_range=1025 65535
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - appwrite-config:/storage/config:ro
@@ -93,6 +102,7 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_DB_MAX_CONNECTIONS
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -187,6 +197,7 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_DB_MAX_CONNECTIONS
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -611,7 +622,7 @@ services:
       - MYSQL_DATABASE=${_APP_DB_SCHEMA}
       - MYSQL_USER=${_APP_DB_USER}
       - MYSQL_PASSWORD=${_APP_DB_PASS}
-    command: 'mysqld --innodb-flush-method=fsync'
+    command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
 
   redis:
     image: redis:7.0.4-alpine

composer.json

@@ -43,26 +43,26 @@
         "ext-sockets": "*",
         "appwrite/php-clamav": "1.1.*",
         "appwrite/php-runtimes": "0.11.*",
-        "utopia-php/platform": "0.3.*",
-        "utopia-php/framework": "0.25.*",
-        "utopia-php/logger": "0.3.*",
         "utopia-php/abuse": "0.16.*",
         "utopia-php/analytics": "0.2.*",
-        "utopia-php/cache": "0.8.*",
         "utopia-php/audit": "0.17.*",
+        "utopia-php/cache": "0.8.*",
         "utopia-php/cli": "0.14.*",
         "utopia-php/config": "0.2.*",
         "utopia-php/database": "0.28.*",
-        "utopia-php/locale": "0.4.*",
-        "utopia-php/registry": "0.5.*",
-        "utopia-php/preloader": "0.2.*",
         "utopia-php/domains": "1.1.*",
-        "utopia-php/swoole": "0.5.*",
-        "utopia-php/storage": "0.11.*",
-        "utopia-php/websocket": "0.1.0",
+        "utopia-php/framework": "0.25.*",
         "utopia-php/image": "0.5.*",
-        "utopia-php/orchestration": "0.7.*",
+        "utopia-php/locale": "0.4.*",
+        "utopia-php/logger": "0.3.*",
+        "utopia-php/orchestration": "0.9.*",
+        "utopia-php/platform": "0.3.*",
         "utopia-php/pools": "0.4.*",
+        "utopia-php/preloader": "0.2.*",
+        "utopia-php/registry": "dev-feat-allow-params as 0.5.0",
+        "utopia-php/storage": "0.11.*",
+        "utopia-php/swoole": "0.5.*",
+        "utopia-php/websocket": "0.1.0",
         "resque/php-resque": "1.3.6",
         "matomo/device-detector": "6.0.0",
         "dragonmantank/cron-expression": "3.3.1",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "bd99fe9f008b3bf3a68b1e3d0b515cc9",
+    "content-hash": "b125039c64ae4cbe0d2a1b57322d0ebe",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -2157,16 +2157,16 @@
         },
         {
             "name": "utopia-php/orchestration",
-            "version": "0.7.0",
+            "version": "0.9.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/orchestration.git",
-                "reference": "484df2c9275a77722fe05f3630cf0e346dcb372e"
+                "reference": "1d4f66684b8c4927f31b695817eae6d84aafd172"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/484df2c9275a77722fe05f3630cf0e346dcb372e",
-                "reference": "484df2c9275a77722fe05f3630cf0e346dcb372e",
+                "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/1d4f66684b8c4927f31b695817eae6d84aafd172",
+                "reference": "1d4f66684b8c4927f31b695817eae6d84aafd172",
                 "shasum": ""
             },
             "require": {
@@ -2200,9 +2200,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/orchestration/issues",
-                "source": "https://github.com/utopia-php/orchestration/tree/0.7.0"
+                "source": "https://github.com/utopia-php/orchestration/tree/0.9.0"
             },
-            "time": "2022-10-28T07:23:38+00:00"
+            "time": "2022-11-09T17:38:00+00:00"
         },
         {
             "name": "utopia-php/platform",
@@ -2255,24 +2255,20 @@
         },
         {
             "name": "utopia-php/pools",
-            "version": "0.4.0",
+            "version": "0.4.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/pools.git",
-                "reference": "63cf2c32f59675ce9b31619ddd618d0b217e423f"
+                "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/pools/zipball/63cf2c32f59675ce9b31619ddd618d0b217e423f",
-                "reference": "63cf2c32f59675ce9b31619ddd618d0b217e423f",
+                "url": "https://api.github.com/repos/utopia-php/pools/zipball/c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979",
+                "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979",
                 "shasum": ""
             },
             "require": {
-                "ext-mongodb": "*",
-                "ext-pdo": "*",
-                "ext-redis": "*",
-                "php": ">=8.0",
-                "utopia-php/cli": "^0.14.0"
+                "php": ">=8.0"
             },
             "require-dev": {
                 "laravel/pint": "1.2.*",
@@ -2304,9 +2300,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/pools/issues",
-                "source": "https://github.com/utopia-php/pools/tree/0.4.0"
+                "source": "https://github.com/utopia-php/pools/tree/0.4.1"
             },
-            "time": "2022-11-10T09:38:57+00:00"
+            "time": "2022-11-15T08:55:16+00:00"
         },
         {
             "name": "utopia-php/preloader",
@@ -2363,16 +2359,16 @@
         },
         {
             "name": "utopia-php/registry",
-            "version": "0.5.0",
+            "version": "dev-feat-allow-params",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/registry.git",
-                "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d"
+                "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d",
-                "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d",
+                "url": "https://api.github.com/repos/utopia-php/registry/zipball/6c571f8f4127094b3af8909d1b595fd6b937255d",
+                "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d",
                 "shasum": ""
             },
             "require": {
@@ -2409,9 +2405,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/registry/issues",
-                "source": "https://github.com/utopia-php/registry/tree/0.5.0"
+                "source": "https://github.com/utopia-php/registry/tree/feat-allow-params"
             },
-            "time": "2021-03-10T10:45:22+00:00"
+            "time": "2022-11-09T02:23:35+00:00"
         },
         {
             "name": "utopia-php/storage",
@@ -5219,9 +5215,18 @@
             "time": "2022-09-28T08:42:51+00:00"
         }
     ],
-    "aliases": [],
+    "aliases": [
+        {
+            "package": "utopia-php/registry",
+            "version": "dev-feat-allow-params",
+            "alias": "0.5.0",
+            "alias_normalized": "0.5.0.0"
+        }
+    ],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "utopia-php/registry": 20
+    },
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": {
@@ -5245,5 +5250,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.2.0"
+    "plugin-api-version": "2.3.0"
 }

docker-compose.yml

@@ -14,7 +14,7 @@ version: '3'
 
 services:
   traefik:
-    image: traefik:2.7
+    image: traefik:2.9
     <<: *x-logging
     container_name: appwrite-traefik
     command:
@@ -33,6 +33,15 @@ services:
       - 8080:80
       - 443:443
       - 9500:8080
+    ulimits:
+      nofile:
+        soft: 655350
+        hard: 655350
+    sysctls:
+      - net.core.somaxconn=1024
+      - net.ipv4.tcp_rmem=1024 4096 16384
+      - net.ipv4.tcp_wmem=1024 4096 16384
+      - net.ipv4.ip_local_port_range=1025 65535
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - appwrite-config:/storage/config:ro
@@ -114,6 +123,7 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_DB_MAX_CONNECTIONS
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -218,6 +228,7 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_DB_MAX_CONNECTIONS
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -730,8 +741,7 @@ services:
       - MYSQL_DATABASE=${_APP_DB_SCHEMA}
       - MYSQL_USER=${_APP_DB_USER}
       - MYSQL_PASSWORD=${_APP_DB_PASS}
-    command: 'mysqld --innodb-flush-method=fsync' # add ' --query_cache_size=0' for DB tests
-    # command: mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile0.bu && mv /var/lib/mysql/ib_logfile1 /var/lib/mysql/ib_logfile1.bu
+    command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
 
   # smtp:
   #   image: appwrite/smtp:1.2.0