diff --git a/.env b/.env index 390e838d8f..5f02020f10 100644 --- a/.env +++ b/.env @@ -23,7 +23,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_DB_MAX_CONNECTIONS=251 +_APP_CONNECTIONS_MAX=251 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 diff --git a/CHANGES.md b/CHANGES.md index 62c0203d12..d111ba226c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ # TBD - Replace Appwrite executor with OpenRuntimes Executor [#4650](https://github.com/appwrite/appwrite/pull/4650) -- Add `_APP_DB_MAX_CONNECTIONS` env var [#4673](https://github.com/appwrite/appwrite/pull/4673) +- Add `_APP_CONNECTIONS_MAX` env var [#4673](https://github.com/appwrite/appwrite/pull/4673) - Update Traefik 2.7 -> 2.9 [#4673](https://github.com/appwrite/appwrite/pull/4673) - Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673) diff --git a/app/config/variables.php b/app/config/variables.php index 96937d9ba5..5c80efa0ef 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -307,7 +307,7 @@ return [ 'filter' => 'password' ], [ - 'name' => '_APP_DB_MAX_CONNECTIONS', + 'name' => '_APP_CONNECTIONS_MAX', 'description' => 'MariaDB server maximum connections.', 'introduction' => 'TBD', 'default' => 251, @@ -315,24 +315,24 @@ return [ 'question' => '', 'filter' => '' ], - [ - 'name' => '_APP_CONNECTIONS_DB_PROJECT', - 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', - 'introduction' => 'TBD', - 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - 'required' => true, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_CONNECTIONS_DB_CONSOLE', - 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', - 'introduction' => 'TBD', - 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - 'required' => true, - 'question' => '', - 'filter' => '' - ] +// [ +// 'name' => '_APP_CONNECTIONS_DB_PROJECT', +// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', +// 'introduction' => 'TBD', +// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', +// 'required' => true, +// 'question' => '', +// 'filter' => '' +// ], +// [ +// 'name' => '_APP_CONNECTIONS_DB_CONSOLE', +// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', +// 'introduction' => 'TBD', +// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', +// 'required' => true, +// 'question' => '', +// 'filter' => '' +// ] ], ], [ diff --git a/app/init.php b/app/init.php index af87960325..ac30673de8 100644 --- a/app/init.php +++ b/app/init.php @@ -97,7 +97,6 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 501; const APP_VERSION_STABLE = '1.0.3'; -const APP_DEFAULT_POOL_SIZE = 64; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -497,7 +496,7 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); -$register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) { +$register->set('pools', function () { $group = new Group(); $fallbackForDB = AppwriteURL::unparse([ @@ -548,6 +547,17 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) { ], ]; + $instances = 2; // REST, Realtime + $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); + $maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251); + $instanceConnections = $maxConnections / $instances; + + if ($workerCount > $instanceConnections) { + throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); + } + + $poolSize = (int)($instanceConnections / $workerCount); + foreach ($connections as $key => $connection) { $type = $connection['type'] ?? ''; $dsns = $connection['dsns'] ?? ''; @@ -621,7 +631,7 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) { break; } - $pool = new Pool($name, $size, function () use ($type, $resource, $dsn) { + $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { // Get Adapter $adapter = null; @@ -1123,26 +1133,6 @@ function getDevice($root): Device } } -/** - * Get database connection pool size for worker processes. - * - * @return int - * @throws \Exception - */ -function getWorkerPoolSize(): int -{ - $reservedConnections = APP_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API - $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); - $maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 251); - $workerConnections = $maxConnections - $reservedConnections; - - if ($workerCount > $workerConnections) { - throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); - } - - return (int)($workerConnections / $workerCount); -} - App::setResource('mode', function ($request) { /** @var Appwrite\Utopia\Request $request */ diff --git a/app/realtime.php b/app/realtime.php index 25c0fee858..54469ebf77 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -37,7 +37,7 @@ function getConsoleDB(): Database global $register; /** @var \Utopia\Pools\Group $pools */ - $pools = $register->get('pools', args: [getWorkerPoolSize()]); + $pools = $register->get('pools'); $dbAdapter = $pools ->get('console') @@ -57,7 +57,7 @@ function getProjectDB(Document $project): Database global $register; /** @var \Utopia\Pools\Group $pools */ - $pools = $register->get('pools', args: [getWorkerPoolSize()]); + $pools = $register->get('pools'); if ($project->isEmpty() || $project->getId() === 'console') { return getConsoleDB(); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 0b83a12729..331b23adb4 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -102,7 +102,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_DB_MAX_CONNECTIONS + - _APP_CONNECTIONS_MAX - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -197,7 +197,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_DB_MAX_CONNECTIONS + - _APP_CONNECTIONS_MAX - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -622,7 +622,7 @@ services: - MYSQL_DATABASE=${_APP_DB_SCHEMA} - MYSQL_USER=${_APP_DB_USER} - MYSQL_PASSWORD=${_APP_DB_PASS} - command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}' + command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}' redis: image: redis:7.0.4-alpine diff --git a/composer.json b/composer.json index 36aed0d96b..3dd374b8e1 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/platform": "0.3.*", "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", - "utopia-php/registry": "dev-feat-allow-params as 0.5.0", + "utopia-php/registry": "0.5.0", "utopia-php/storage": "0.11.*", "utopia-php/swoole": "0.5.*", "utopia-php/websocket": "0.1.0", diff --git a/composer.lock b/composer.lock index f042279e4f..2e39916a0c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b125039c64ae4cbe0d2a1b57322d0ebe", + "content-hash": "bf3e2ed6ee8e49ab74af97b368b89a63", "packages": [ { "name": "adhocore/jwt", @@ -2359,16 +2359,16 @@ }, { "name": "utopia-php/registry", - "version": "dev-feat-allow-params", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/registry.git", - "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d" + "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/registry/zipball/6c571f8f4127094b3af8909d1b595fd6b937255d", - "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d", + "url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d", + "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d", "shasum": "" }, "require": { @@ -2405,9 +2405,9 @@ ], "support": { "issues": "https://github.com/utopia-php/registry/issues", - "source": "https://github.com/utopia-php/registry/tree/feat-allow-params" + "source": "https://github.com/utopia-php/registry/tree/0.5.0" }, - "time": "2022-11-09T02:23:35+00:00" + "time": "2021-03-10T10:45:22+00:00" }, { "name": "utopia-php/storage", @@ -5215,18 +5215,9 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [ - { - "package": "utopia-php/registry", - "version": "dev-feat-allow-params", - "alias": "0.5.0", - "alias_normalized": "0.5.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/registry": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 4bef6b1a01..8f0576aae4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,7 +123,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_DB_MAX_CONNECTIONS + - _APP_CONNECTIONS_MAX - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -229,7 +229,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_DB_MAX_CONNECTIONS + - _APP_CONNECTIONS_MAX - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -752,7 +752,7 @@ services: - MYSQL_DATABASE=${_APP_DB_SCHEMA} - MYSQL_USER=${_APP_DB_USER} - MYSQL_PASSWORD=${_APP_DB_PASS} - command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}' + command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}' # smtp: # image: appwrite/smtp:1.2.0