From a6ffab09df3d69ce2c5c73a2b1de22418a5fac45 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 7 Nov 2022 23:31:54 +1300 Subject: [PATCH 01/25] WIP fix too many connections --- app/realtime.php | 13 ++++++++----- docker-compose.yml | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index be87c3d6e6..8ee4e64761 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -35,6 +35,9 @@ Runtime::enableCoroutine(SWOOLE_HOOK_ALL); $realtime = new Realtime(); +$dbPool = $register->get('dbPool'); +$redisPool = $register->get('redisPool'); + /** * Table for statistics across all workers. */ @@ -354,15 +357,15 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, Console::error('Failed to restart pub/sub...'); }); -$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $register, $stats, &$realtime, $logError) { +$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $dbPool, $redisPool, $stats, &$realtime, $logError) { $app = new App('UTC'); $request = new Request($request); $response = new Response(new SwooleResponse()); /** @var PDO $db */ - $db = $register->get('dbPool')->get(); + $db = $dbPool->get(); /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); + $redis = $redisPool->get(); Console::info("Connection open (user: {$connection})"); @@ -477,8 +480,8 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, /** * Put used PDO and Redis Connections back into their pools. */ - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + $dbPool->put($db); + $redisPool->put($redis); } }); diff --git a/docker-compose.yml b/docker-compose.yml index ea9241a9d3..b5b6a271c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -707,7 +707,7 @@ services: - MYSQL_DATABASE=${_APP_DB_SCHEMA} - MYSQL_USER=${_APP_DB_USER} - MYSQL_PASSWORD=${_APP_DB_PASS} - command: 'mysqld --innodb-flush-method=fsync' # add ' --query_cache_size=0' for DB tests + command: 'mysqld --innodb-flush-method=fsync --max_connections=49152' # 128 (CPUs) * 6 (Workers) * 64 (Pool size) # command: mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile0.bu && mv /var/lib/mysql/ib_logfile1 /var/lib/mysql/ib_logfile1.bu # smtp: From 11c306068290f3221a163da8ed5c89bc62306f81 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 8 Nov 2022 12:25:54 +1300 Subject: [PATCH 02/25] Revert "WIP fix too many connections" registry changes This reverts commit a6ffab09df3d69ce2c5c73a2b1de22418a5fac45. --- app/realtime.php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 8ee4e64761..be87c3d6e6 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -35,9 +35,6 @@ Runtime::enableCoroutine(SWOOLE_HOOK_ALL); $realtime = new Realtime(); -$dbPool = $register->get('dbPool'); -$redisPool = $register->get('redisPool'); - /** * Table for statistics across all workers. */ @@ -357,15 +354,15 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, Console::error('Failed to restart pub/sub...'); }); -$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $dbPool, $redisPool, $stats, &$realtime, $logError) { +$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $register, $stats, &$realtime, $logError) { $app = new App('UTC'); $request = new Request($request); $response = new Response(new SwooleResponse()); /** @var PDO $db */ - $db = $dbPool->get(); + $db = $register->get('dbPool')->get(); /** @var Redis $redis */ - $redis = $redisPool->get(); + $redis = $register->get('redisPool')->get(); Console::info("Connection open (user: {$connection})"); @@ -480,8 +477,8 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, /** * Put used PDO and Redis Connections back into their pools. */ - $dbPool->put($db); - $redisPool->put($redis); + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); } }); From eef2fdb72c30b0384eb4326ce5b03b9bec0ecba7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 8 Nov 2022 21:33:13 +1300 Subject: [PATCH 03/25] Allow up to 65K realtime connections through Traefik --- docker-compose.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b5b6a271c2..f72115c0be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ version: '3' services: traefik: - image: traefik:2.7 + image: traefik:2.9 <<: *x-logging container_name: appwrite-traefik command: @@ -33,6 +33,17 @@ services: - 8080:80 - 443:443 - 9500:8080 + ulimits: + nproc: 65535 + nofile: + soft: 65535 + hard: 65535 + sysctls: + - net.core.somaxconn=1024 + - net.ipv4.tcp_rmem=1024 4096 16384 + - net.ipv4.tcp_wmem=1024 4096 16384 + - net.ipv4.tcp_moderate_rcvbuf=0 + - net.ipv4.ip_local_port_range=1025 65535 volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-config:/storage/config:ro From 958e5424f024720e050274aa3b243e0a23476e6f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 9 Nov 2022 16:12:50 +1300 Subject: [PATCH 04/25] Revert "Revert "WIP fix too many connections" registry changes" This reverts commit 11c306068290f3221a163da8ed5c89bc62306f81. --- app/realtime.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index be87c3d6e6..8ee4e64761 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -35,6 +35,9 @@ Runtime::enableCoroutine(SWOOLE_HOOK_ALL); $realtime = new Realtime(); +$dbPool = $register->get('dbPool'); +$redisPool = $register->get('redisPool'); + /** * Table for statistics across all workers. */ @@ -354,15 +357,15 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, Console::error('Failed to restart pub/sub...'); }); -$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $register, $stats, &$realtime, $logError) { +$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $dbPool, $redisPool, $stats, &$realtime, $logError) { $app = new App('UTC'); $request = new Request($request); $response = new Response(new SwooleResponse()); /** @var PDO $db */ - $db = $register->get('dbPool')->get(); + $db = $dbPool->get(); /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); + $redis = $redisPool->get(); Console::info("Connection open (user: {$connection})"); @@ -477,8 +480,8 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, /** * Put used PDO and Redis Connections back into their pools. */ - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + $dbPool->put($db); + $redisPool->put($redis); } }); From e1d80c2b93c0027bc4539508c5a856296435b396 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 9 Nov 2022 16:19:01 +1300 Subject: [PATCH 05/25] Add get worker pool size function --- app/init.php | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/app/init.php b/app/init.php index 054721d439..cc979e6c76 100644 --- a/app/init.php +++ b/app/init.php @@ -1006,6 +1006,26 @@ function getDevice($root): Device } } +/** + * Get database connection pool size for worker processes. + * + * @return int + * @throws \Exception + */ +function getWorkerPoolSize(): int +{ + $reservedConnections = APP_DATABASE_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API + $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); + $maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 1001); + $workerConnections = $maxConnections - $reservedConnections; + + if ($workerCount > $workerConnections) { + throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.'); + } + + return (int)($workerConnections / $workerCount); +} + App::setResource('mode', function ($request) { /** @var Appwrite\Utopia\Request $request */ From fafbe71b24e6de4c28c456946f0cd3f080c4885c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 9 Nov 2022 16:21:27 +1300 Subject: [PATCH 06/25] Accept a size parameter when creating db/redis pools --- app/init.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/init.php b/app/init.php index cc979e6c76..16dac440ac 100644 --- a/app/init.php +++ b/app/init.php @@ -104,6 +104,7 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url'; const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange'; const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange'; const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1073741824; // 2^32 bits / 4 bits per char +const APP_DATABASE_DEFAULT_POOL_SIZE = 64; const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; @@ -495,7 +496,7 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); -$register->set('dbPool', function () { +$register->set('dbPool', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { // Register DB connection $dbHost = App::getEnv('_APP_DB_HOST', ''); $dbPort = App::getEnv('_APP_DB_PORT', ''); @@ -519,12 +520,12 @@ $register->set('dbPool', function () { PDO::ATTR_EMULATE_PREPARES => true, PDO::ATTR_STRINGIFY_FETCHES => true, ]), - 64 + $size ); return $pool; }); -$register->set('redisPool', function () { +$register->set('redisPool', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { $redisHost = App::getEnv('_APP_REDIS_HOST', ''); $redisPort = App::getEnv('_APP_REDIS_PORT', ''); $redisUser = App::getEnv('_APP_REDIS_USER', ''); @@ -541,7 +542,7 @@ $register->set('redisPool', function () { ->withPort($redisPort) ->withAuth($redisAuth) ->withDbIndex(0), - 64 + $size ); return $pool; From e901be28decbece82eb36c295555f9537ccc30a5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 9 Nov 2022 16:22:03 +1300 Subject: [PATCH 07/25] Add max connections env var --- .env | 1 + 1 file changed, 1 insertion(+) diff --git a/.env b/.env index 65fb54cb04..bc2b32cde8 100644 --- a/.env +++ b/.env @@ -23,6 +23,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword +_APP_DB_MAX_CONNECTIONS=1001 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= From 4fe2289e7fdcdbbff5b1949ccea4a115aac02581 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 9 Nov 2022 16:24:42 +1300 Subject: [PATCH 08/25] Add max connections variable --- app/config/variables.php | 9 +++ composer.json | 2 +- composer.lock | 128 ++++++++++++++++++++++++++++++--------- docker-compose.yml | 2 +- 4 files changed, 110 insertions(+), 31 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 9f3bc018e8..ba6aa9a14a 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -306,6 +306,15 @@ return [ 'question' => '', 'filter' => 'password' ], + [ + 'name' => '_APP_DB_MAX_CONNECTIONS', + 'description' => 'MariaDB server maximum connections.', + 'introduction' => '1.2.0', + 'default' => 1001, + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ diff --git a/composer.json b/composer.json index 05baa8f008..6a80960aec 100644 --- a/composer.json +++ b/composer.json @@ -53,7 +53,7 @@ "utopia-php/config": "0.2.*", "utopia-php/database": "0.28.*", "utopia-php/locale": "0.4.*", - "utopia-php/registry": "0.5.*", + "utopia-php/registry": "dev-feat-allow-params as 0.5.0", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.3.*", diff --git a/composer.lock b/composer.lock index c0320c77da..3d29f7563a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "51f81d435f4b5b7a9a6ea8f81b470353", + "content-hash": "6c9a067a247f45ee7c5c224577201d75", "packages": [ { "name": "adhocore/jwt", @@ -115,15 +115,15 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.11.0", + "version": "0.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "547fc026e11c0946846a8ac690898f5bf53be101" + "reference": "9d74a477ba3333cbcfac565c46fcf19606b7b603" }, "require": { "php": ">=8.0", - "utopia-php/system": "0.4.*" + "utopia-php/system": "0.6.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -154,7 +154,7 @@ "php", "runtimes" ], - "time": "2022-08-15T14:03:36+00:00" + "time": "2022-11-07T16:45:52+00:00" }, { "name": "chillerlan/php-qrcode", @@ -300,16 +300,16 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.13.1", + "version": "v1.14.0", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49" + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/dccc8a46586475075fbb012d8bd523b8a938c2dc", + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc", "shasum": "" }, "require": { @@ -341,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1" + "source": "https://github.com/colinmollenhour/credis/tree/v1.14.0" }, - "time": "2022-06-20T22:56:59+00:00" + "time": "2022-11-09T01:18:39+00:00" }, { "name": "dragonmantank/cron-expression", @@ -803,6 +803,72 @@ }, "time": "2020-12-26T17:45:17+00:00" }, + { + "name": "laravel/pint", + "version": "v1.2.0", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.11.0", + "illuminate/view": "^9.27", + "laravel-zero/framework": "^9.1.3", + "mockery/mockery": "^1.5.0", + "nunomaduro/larastan": "^2.2", + "nunomaduro/termwind": "^1.14.0", + "pestphp/pest": "^1.22.1" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2022-09-13T15:07:15+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -2205,16 +2271,16 @@ }, { "name": "utopia-php/registry", - "version": "0.5.0", + "version": "dev-feat-allow-params", "source": { "type": "git", "url": "https://github.com/utopia-php/registry.git", - "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d" + "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d", - "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d", + "url": "https://api.github.com/repos/utopia-php/registry/zipball/6c571f8f4127094b3af8909d1b595fd6b937255d", + "reference": "6c571f8f4127094b3af8909d1b595fd6b937255d", "shasum": "" }, "require": { @@ -2251,9 +2317,9 @@ ], "support": { "issues": "https://github.com/utopia-php/registry/issues", - "source": "https://github.com/utopia-php/registry/tree/0.5.0" + "source": "https://github.com/utopia-php/registry/tree/feat-allow-params" }, - "time": "2021-03-10T10:45:22+00:00" + "time": "2022-11-09T02:23:35+00:00" }, { "name": "utopia-php/storage", @@ -2368,23 +2434,25 @@ }, { "name": "utopia-php/system", - "version": "0.4.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0" + "reference": "289c4327713deadc9c748b5317d248133a02f245" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0", + "url": "https://api.github.com/repos/utopia-php/system/zipball/289c4327713deadc9c748b5317d248133a02f245", + "reference": "289c4327713deadc9c748b5317d248133a02f245", "shasum": "" }, "require": { + "laravel/pint": "1.2.*", "php": ">=7.4" }, "require-dev": { "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", "vimeo/psalm": "4.0.1" }, "type": "library", @@ -2417,9 +2485,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.4.0" + "source": "https://github.com/utopia-php/system/tree/0.6.0" }, - "time": "2021-02-04T14:14:49+00:00" + "time": "2022-11-07T13:51:59+00:00" }, { "name": "utopia-php/websocket", @@ -5066,14 +5134,16 @@ ], "aliases": [ { - "package": "utopia-php/database", - "version": "0.28.0.0", - "alias": "0.26.99", - "alias_normalized": "0.26.99.0" + "package": "utopia-php/registry", + "version": "dev-feat-allow-params", + "alias": "0.5.0", + "alias_normalized": "0.5.0.0" } ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/registry": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5097,5 +5167,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/docker-compose.yml b/docker-compose.yml index f72115c0be..664da4a959 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -718,7 +718,7 @@ services: - MYSQL_DATABASE=${_APP_DB_SCHEMA} - MYSQL_USER=${_APP_DB_USER} - MYSQL_PASSWORD=${_APP_DB_PASS} - command: 'mysqld --innodb-flush-method=fsync --max_connections=49152' # 128 (CPUs) * 6 (Workers) * 64 (Pool size) + command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}' # command: mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile0.bu && mv /var/lib/mysql/ib_logfile1 /var/lib/mysql/ib_logfile1.bu # smtp: From b6839fa8f1e0d52c7785cd292467872f8a54dbab Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 9 Nov 2022 16:39:15 +1300 Subject: [PATCH 09/25] Capture pools on init --- app/realtime.php | 52 +++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 8ee4e64761..a346cb4dc8 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -4,6 +4,7 @@ use Appwrite\Auth\Auth; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Response; +use Swoole\ConnectionPool; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Runtime; @@ -35,7 +36,7 @@ Runtime::enableCoroutine(SWOOLE_HOOK_ALL); $realtime = new Realtime(); -$dbPool = $register->get('dbPool'); +$dbPool = $register->get('dbPool', args: [getWorkerPoolSize()]); $redisPool = $register->get('redisPool'); /** @@ -98,7 +99,7 @@ $logError = function (Throwable $error, string $action) use ($register) { $server->error($logError); -function getDatabase(Registry &$register, string $namespace) +function getDatabase(ConnectionPool $dbPool, ConnectionPool $redisPool, string $namespace) { $attempts = 0; @@ -106,8 +107,8 @@ function getDatabase(Registry &$register, string $namespace) try { $attempts++; - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); + $db = $dbPool->get(); + $redis = $redisPool->get(); $cache = new Cache(new RedisCache($redis)); $database = new Database(new MariaDB($db), $cache); @@ -130,23 +131,23 @@ function getDatabase(Registry &$register, string $namespace) return [ $database, - function () use ($register, $db, $redis) { - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + function () use ($dbPool, $redisPool, $db, $redis) { + $dbPool->put($db); + $redisPool->put($redis); } ]; } -$server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) { +$server->onStart(function () use ($stats, $dbPool, $redisPool, $containerId, &$statsDocument, $logError) { sleep(5); // wait for the initial database schema to be ready Console::success('Server started successfully'); /** * Create document for this worker to share stats across Containers. */ - go(function () use ($register, $containerId, &$statsDocument) { + go(function () use ($dbPool, $redisPool, $containerId, &$statsDocument) { $attempts = 0; - [$database, $returnDatabase] = getDatabase($register, '_console'); + [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console'); do { try { $attempts++; @@ -172,7 +173,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume /** * Save current connections to the Database every 5 seconds. */ - Timer::tick(5000, function () use ($register, $stats, &$statsDocument, $logError) { + Timer::tick(5000, function () use ($dbPool, $redisPool, $stats, &$statsDocument, $logError) { $payload = []; foreach ($stats as $projectId => $value) { $payload[$projectId] = $stats->get($projectId, 'connectionsTotal'); @@ -182,7 +183,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } try { - [$database, $returnDatabase] = getDatabase($register, '_console'); + [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console'); $statsDocument ->setAttribute('timestamp', DateTime::now()) @@ -197,18 +198,18 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume }); }); -$server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $logError) { +$server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPool, $stats, $realtime, $logError) { Console::success('Worker ' . $workerId . ' started successfully'); $attempts = 0; $start = time(); - Timer::tick(5000, function () use ($server, $register, $realtime, $stats, $logError) { + Timer::tick(5000, function () use ($server, $dbPool, $redisPool, $realtime, $stats, $logError) { /** * Sending current connections to project channels on the console project every 5 seconds. */ if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) { - [$database, $returnDatabase] = getDatabase($register, '_console'); + [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console'); $payload = []; @@ -289,7 +290,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $start = time(); /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); + $redis = $redisPool->get(); $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); if ($redis->ping(true)) { @@ -299,7 +300,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, Console::error('Pub/sub failed (worker: ' . $workerId . ')'); } - $redis->subscribe(['realtime'], function (Redis $redis, string $channel, string $payload) use ($server, $workerId, $stats, $register, $realtime) { + $redis->subscribe(['realtime'], function (Redis $redis, string $channel, string $payload) use ($server, $workerId, $stats, $dbPool, $redisPool, $realtime) { $event = json_decode($payload, true); if ($event['permissionsChanged'] && isset($event['userId'])) { @@ -308,9 +309,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) { $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId])); - [$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, '_console'); + [$consoleDatabase, $returnConsoleDatabase] = getDatabase($dbPool, $redisPool, '_console'); $project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId)); - [$database, $returnDatabase] = getDatabase($register, "_{$project->getInternalId()}"); + [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, "_{$project->getInternalId()}"); $user = $database->getDocument('users', $userId); @@ -347,10 +348,11 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, call_user_func($logError, $th, "pubSubConnection"); Console::error('Pub/sub error: ' . $th->getMessage()); - $register->get('redisPool')->put($redis); $attempts++; sleep(DATABASE_RECONNECT_SLEEP); continue; + } finally { + $redisPool->put($redis); } } @@ -485,11 +487,11 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, } }); -$server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) { +$server->onMessage(function (int $connection, string $message) use ($server, $dbPool, $redisPool, $realtime, $containerId) { try { $response = new Response(new SwooleResponse()); - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); + $db = $dbPool->get(); + $redis = $redisPool->get(); $cache = new Cache(new RedisCache($redis)); $database = new Database(new MariaDB($db), $cache); @@ -583,8 +585,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $server->close($connection, $th->getCode()); } } finally { - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + $dbPool->put($db); + $redisPool->put($redis); } }); From d3ea95b60e5a9caf314453a51c3028a57d3183fd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 10 Nov 2022 19:57:05 +1300 Subject: [PATCH 10/25] Remove uneccessary process limits --- app/tasks/volume-sync.php | 1 - docker-compose.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/app/tasks/volume-sync.php b/app/tasks/volume-sync.php index 2114348b1b..fe0eee07b9 100644 --- a/app/tasks/volume-sync.php +++ b/app/tasks/volume-sync.php @@ -36,6 +36,5 @@ $cli Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr); Console::success($stdout); Console::error($stderr); - }, $interval); }); diff --git a/docker-compose.yml b/docker-compose.yml index ed0e36f4ca..44b8c83034 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,6 @@ services: - 443:443 - 9500:8080 ulimits: - nproc: 65535 nofile: soft: 65535 hard: 65535 From def7fb71f16f19cb012c7ec323dd997a9c6c5637 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 10 Nov 2022 20:01:18 +1300 Subject: [PATCH 11/25] Increase Traefik files limits --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 44b8c83034..7fdf459a45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,8 +35,8 @@ services: - 9500:8080 ulimits: nofile: - soft: 65535 - hard: 65535 + soft: 655350 + hard: 655350 sysctls: - net.core.somaxconn=1024 - net.ipv4.tcp_rmem=1024 4096 16384 From 3a6ed84ce7e5dc90f16e5580821c3d260f5db9ee Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Nov 2022 11:22:47 +1300 Subject: [PATCH 12/25] Update CLI, pool, orchestration --- composer.json | 24 +++++------ composer.lock | 107 ++++++++++++++++++++++++-------------------------- 2 files changed, 64 insertions(+), 67 deletions(-) diff --git a/composer.json b/composer.json index 91c561d7d1..2c44d16527 100644 --- a/composer.json +++ b/composer.json @@ -43,25 +43,25 @@ "ext-sockets": "*", "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", - "utopia-php/framework": "0.23.*", - "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.16.*", "utopia-php/analytics": "0.2.*", - "utopia-php/cache": "0.8.*", "utopia-php/audit": "0.17.*", - "utopia-php/cli": "0.13.*", + "utopia-php/cache": "0.8.*", + "utopia-php/cli": "0.14.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.28.*", - "utopia-php/locale": "0.4.*", - "utopia-php/registry": "dev-feat-allow-params as 0.5.0", - "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", - "utopia-php/swoole": "0.5.*", - "utopia-php/storage": "0.11.*", - "utopia-php/websocket": "0.1.0", + "utopia-php/framework": "0.23.*", "utopia-php/image": "0.5.*", - "utopia-php/orchestration": "0.6.*", - "utopia-php/pools": "dev-feat-optimize-filling as 0.3.0", + "utopia-php/locale": "0.4.*", + "utopia-php/logger": "0.3.*", + "utopia-php/orchestration": "0.9.*", + "utopia-php/pools": "dev-feat-remove-dependencies as 0.4.0", + "utopia-php/preloader": "0.2.*", + "utopia-php/registry": "dev-feat-allow-params as 0.5.0", + "utopia-php/storage": "0.11.*", + "utopia-php/swoole": "0.5.*", + "utopia-php/websocket": "0.1.0", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index b245d26383..49cb7c1f18 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6c9a067a247f45ee7c5c224577201d75", + "content-hash": "164c92299614df333e6ab5a45672bc97", "packages": [ { "name": "adhocore/jwt", @@ -1733,16 +1733,16 @@ }, { "name": "utopia-php/cli", - "version": "0.13.0", + "version": "0.14.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "69e68f8ed525fe162fae950a0507ed28a0f179bc" + "reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/69e68f8ed525fe162fae950a0507ed28a0f179bc", - "reference": "69e68f8ed525fe162fae950a0507ed28a0f179bc", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/c30ef985a4e739758a0d95eb0706b357b6d8c086", + "reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086", "shasum": "" }, "require": { @@ -1751,7 +1751,7 @@ }, "require-dev": { "phpunit/phpunit": "^9.3", - "vimeo/psalm": "4.0.1" + "squizlabs/php_codesniffer": "^3.6" }, "type": "library", "autoload": { @@ -1780,9 +1780,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.13.0" + "source": "https://github.com/utopia-php/cli/tree/0.14.0" }, - "time": "2022-04-26T08:41:22+00:00" + "time": "2022-10-09T10:19:07+00:00" }, { "name": "utopia-php/config", @@ -2157,21 +2157,21 @@ }, { "name": "utopia-php/orchestration", - "version": "0.6.0", + "version": "0.9.0", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "94263976413871efb6b16157a7101a81df3b6d78" + "reference": "1d4f66684b8c4927f31b695817eae6d84aafd172" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/94263976413871efb6b16157a7101a81df3b6d78", - "reference": "94263976413871efb6b16157a7101a81df3b6d78", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/1d4f66684b8c4927f31b695817eae6d84aafd172", + "reference": "1d4f66684b8c4927f31b695817eae6d84aafd172", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/cli": "0.13.*" + "utopia-php/cli": "0.14.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -2187,12 +2187,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "Lite & fast micro PHP abstraction library for container orchestration", "keywords": [ "docker", @@ -2206,34 +2200,31 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/0.6.0" + "source": "https://github.com/utopia-php/orchestration/tree/0.9.0" }, - "time": "2022-07-13T16:47:18+00:00" + "time": "2022-11-09T17:38:00+00:00" }, { "name": "utopia-php/pools", - "version": "dev-feat-optimize-filling", + "version": "dev-feat-remove-dependencies", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "be603898142f55df9db5058f81a610ead7769d7d" + "reference": "bdc023b98023fc6a5978ee99cc750256acaeb194" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/be603898142f55df9db5058f81a610ead7769d7d", - "reference": "be603898142f55df9db5058f81a610ead7769d7d", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/bdc023b98023fc6a5978ee99cc750256acaeb194", + "reference": "bdc023b98023fc6a5978ee99cc750256acaeb194", "shasum": "" }, "require": { - "ext-mongodb": "*", - "ext-pdo": "*", - "ext-redis": "*", - "php": ">=8.0", - "utopia-php/cli": "^0.13.0" + "php": ">=8.0" }, "require-dev": { - "phpunit/phpunit": "^9.4", - "vimeo/psalm": "4.0.1" + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.8.*", + "phpunit/phpunit": "^9.3" }, "type": "library", "autoload": { @@ -2260,9 +2251,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/feat-optimize-filling" + "source": "https://github.com/utopia-php/pools/tree/feat-remove-dependencies" }, - "time": "2022-11-03T18:50:28+00:00" + "time": "2022-11-13T22:14:50+00:00" }, { "name": "utopia-php/preloader", @@ -2954,16 +2945,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.1", + "version": "v4.15.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900" + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", "shasum": "" }, "require": { @@ -3004,9 +2995,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.2" }, - "time": "2022-09-04T07:30:47+00:00" + "time": "2022-11-12T15:38:23+00:00" }, { "name": "phar-io/manifest", @@ -4836,16 +4827,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a", + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a", "shasum": "" }, "require": { @@ -4860,7 +4851,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -4898,7 +4889,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0" }, "funding": [ { @@ -4914,20 +4905,20 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534", + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534", "shasum": "" }, "require": { @@ -4942,7 +4933,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -4981,7 +4972,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0" }, "funding": [ { @@ -4997,7 +4988,7 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "textalk/websocket", @@ -5176,6 +5167,12 @@ } ], "aliases": [ + { + "package": "utopia-php/pools", + "version": "dev-feat-remove-dependencies", + "alias": "0.4.0", + "alias_normalized": "0.4.0.0" + }, { "package": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5185,7 +5182,7 @@ ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/pools": 20 + "utopia-php/pools": 20, "utopia-php/registry": 20 }, "prefer-stable": false, From 6d1ae05b64db5a2bd31a4497559f24c3af5acbd1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Nov 2022 11:42:35 +1300 Subject: [PATCH 13/25] Fix registry usages --- app/realtime.php | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 4771669476..9b7cb0c87e 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -37,7 +37,8 @@ function getConsoleDB(): Database { global $register; - $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + /** @var \Utopia\Pools\Group $pools */ + $pools = $register->get('pools', args: [getWorkerPoolSize()]); $dbAdapter = $pools ->get('console') @@ -56,7 +57,8 @@ function getProjectDB(Document $project): Database { global $register; - $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + /** @var \Utopia\Pools\Group $pools */ + $pools = $register->get('pools', args: [getWorkerPoolSize()]); if ($project->isEmpty() || $project->getId() === 'console') { return getConsoleDB(); @@ -96,9 +98,6 @@ function getCache(): Cache $realtime = new Realtime(); -$dbPool = $register->get('dbPool', args: [getWorkerPoolSize()]); -$redisPool = $register->get('redisPool'); - /** * Table for statistics across all workers. */ @@ -166,7 +165,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume /** * Create document for this worker to share stats across Containers. */ - go(function () use ($dbPool, $redisPool, $containerId, &$statsDocument) { + go(function () use ($register, $containerId, &$statsDocument) { $attempts = 0; $database = getConsoleDB(); @@ -195,7 +194,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume /** * Save current connections to the Database every 5 seconds. */ - Timer::tick(5000, function () use ($dbPool, $redisPool, $stats, &$statsDocument, $logError) { + Timer::tick(5000, function () use ($register, $stats, &$statsDocument, $logError) { $payload = []; foreach ($stats as $projectId => $value) { $payload[$projectId] = $stats->get($projectId, 'connectionsTotal'); @@ -220,13 +219,13 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume }); }); -$server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPool, $stats, $realtime, $logError) { +$server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $logError) { Console::success('Worker ' . $workerId . ' started successfully'); $attempts = 0; $start = time(); - Timer::tick(5000, function () use ($server, $dbPool, $redisPool, $realtime, $stats, $logError) { + Timer::tick(5000, function () use ($server, $register, $realtime, $stats, $logError) { /** * Sending current connections to project channels on the console project every 5 seconds. */ @@ -321,7 +320,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo Console::error('Pub/sub failed (worker: ' . $workerId . ')'); } - $redis->subscribe(['realtime'], function (Redis $redis, string $channel, string $payload) use ($server, $workerId, $stats, $dbPool, $redisPool, $realtime) { + $redis->subscribe(['realtime'], function (Redis $redis, string $channel, string $payload) use ($server, $workerId, $stats, $register, $realtime) { $event = json_decode($payload, true); if ($event['permissionsChanged'] && isset($event['userId'])) { @@ -368,19 +367,18 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo call_user_func($logError, $th, "pubSubConnection"); Console::error('Pub/sub error: ' . $th->getMessage()); - $register->get('pools')->reclaim(); $attempts++; sleep(DATABASE_RECONNECT_SLEEP); continue; } finally { - $redisPool->put($redis); + $register->get('pools')->reclaim(); } } Console::error('Failed to restart pub/sub...'); }); -$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $dbPool, $redisPool, $stats, &$realtime, $logError) { +$server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $register, $stats, &$realtime, $logError) { $app = new App('UTC'); $request = new Request($request); $response = new Response(new SwooleResponse()); @@ -487,7 +485,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, } }); -$server->onMessage(function (int $connection, string $message) use ($server, $dbPool, $redisPool, $realtime, $containerId) { +$server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) { try { $response = new Response(new SwooleResponse()); $projectId = $realtime->connections[$connection]['projectId']; @@ -562,7 +560,6 @@ $server->onMessage(function (int $connection, string $message) use ($server, $db default: throw new Exception('Message type is not valid.', 1003); - break; } } catch (\Throwable $th) { $response = [ From 447f3f6756021d0930d84f8cd669c7392db6a394 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 15:45:18 +1300 Subject: [PATCH 14/25] Add missing env to image --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 82d7b59c04..2dc0bbb66a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -124,6 +124,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_MAX_CONNECTIONS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -230,6 +231,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_MAX_CONNECTIONS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER From 1c24421a95d2ee90fd17f02f8ac8e350b714b9a8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 15:45:53 +1300 Subject: [PATCH 15/25] Use buffered queries --- app/init.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 6d7028e884..920ac7922d 100644 --- a/app/init.php +++ b/app/init.php @@ -598,7 +598,8 @@ $register->set('pools', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true + PDO::ATTR_STRINGIFY_FETCHES => true, + PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, )); }); }; From 7de2f69f38174981754a663b0998a3c08c44005b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 19:43:04 +1300 Subject: [PATCH 16/25] Revert "Use buffered queries" This reverts commit 1c24421a95d2ee90fd17f02f8ac8e350b714b9a8. --- app/init.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 920ac7922d..6d7028e884 100644 --- a/app/init.php +++ b/app/init.php @@ -598,8 +598,7 @@ $register->set('pools', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, + PDO::ATTR_STRINGIFY_FETCHES => true )); }); }; From ada54cacd0b8a4b5a043bc3bfe8538ac826f8c7a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 20:29:27 +1300 Subject: [PATCH 17/25] Update default env values --- .env | 2 +- app/config/variables.php | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.env b/.env index c3785cb49a..973b585b03 100644 --- a/.env +++ b/.env @@ -23,7 +23,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_DB_MAX_CONNECTIONS=1001 +_APP_DB_MAX_CONNECTIONS=251 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mysql:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mysql:3306/appwrite _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 diff --git a/app/config/variables.php b/app/config/variables.php index 7cacd39ca4..fe1aff6c30 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -309,30 +309,30 @@ return [ [ 'name' => '_APP_DB_MAX_CONNECTIONS', 'description' => 'MariaDB server maximum connections.', - 'introduction' => '1.2.0', - 'default' => 1001, + 'introduction' => 'TBD', + 'default' => 251, 'required' => false, 'question' => '', 'filter' => '' ], - // [ - // 'name' => '_APP_CONNECTIONS_DB_PROJECT', - // 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', - // 'introduction' => 'TBD', - // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - // 'required' => true, - // 'question' => '', - // 'filter' => '' - // ], - // [ - // 'name' => '_APP_CONNECTIONS_DB_CONSOLE', - // 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', - // 'introduction' => 'TBD', - // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', - // 'required' => true, - // 'question' => '', - // 'filter' => '' - // ] + [ + 'name' => '_APP_CONNECTIONS_DB_PROJECT', + 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + 'introduction' => 'TBD', + 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + 'required' => true, + 'question' => '', + 'filter' => '' + ] ], ], [ From af391b8f7adc429c7a0883361ed38ad0dc288198 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 20:30:01 +1300 Subject: [PATCH 18/25] Update default pool size name --- app/init.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init.php b/app/init.php index 6d7028e884..8b9020f25d 100644 --- a/app/init.php +++ b/app/init.php @@ -97,6 +97,7 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 501; const APP_VERSION_STABLE = '1.0.3'; +const APP_DEFAULT_POOL_SIZE = 64; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -105,7 +106,6 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url'; const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange'; const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange'; const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1073741824; // 2^32 bits / 4 bits per char -const APP_DATABASE_DEFAULT_POOL_SIZE = 64; const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; @@ -497,7 +497,7 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); -$register->set('pools', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { +$register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) { $group = new Group(); $fallbackForDB = AppwriteURL::unparse([ @@ -1131,7 +1131,7 @@ function getDevice($root): Device */ function getWorkerPoolSize(): int { - $reservedConnections = APP_DATABASE_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API + $reservedConnections = APP_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); $maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 1001); $workerConnections = $maxConnections - $reservedConnections; From c4be0260c3a3cd2637de996c004dfc60b9ecdf57 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 20:36:04 +1300 Subject: [PATCH 19/25] Update env default max connections --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 8b9020f25d..7fd6acdcd1 100644 --- a/app/init.php +++ b/app/init.php @@ -1133,7 +1133,7 @@ function getWorkerPoolSize(): int { $reservedConnections = APP_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); - $maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 1001); + $maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 251); $workerConnections = $maxConnections - $reservedConnections; if ($workerCount > $workerConnections) { From cca33c6c5d38723a1e791d88756c4aeaac7af53d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 20:39:41 +1300 Subject: [PATCH 20/25] Remove redundant import --- app/realtime.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index 9b7cb0c87e..25c0fee858 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -4,7 +4,6 @@ use Appwrite\Auth\Auth; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Response; -use Swoole\ConnectionPool; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Runtime; From e750c1966026cd6e38ed2cddbd19398f55948ec1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 20:52:08 +1300 Subject: [PATCH 21/25] Remove tcp read buffer auto-size disabling --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2dc0bbb66a..5148e85527 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,6 @@ services: - net.core.somaxconn=1024 - net.ipv4.tcp_rmem=1024 4096 16384 - net.ipv4.tcp_wmem=1024 4096 16384 - - net.ipv4.tcp_moderate_rcvbuf=0 - net.ipv4.ip_local_port_range=1025 65535 volumes: - /var/run/docker.sock:/var/run/docker.sock From b8b78adfdc084e4a4f4eb08395f60c8531be9f5f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 21:05:52 +1300 Subject: [PATCH 22/25] Update changelog --- CHANGES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 2c649ab5f0..ed0fc8d8c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +# TBD + +- Add `_APP_DB_MAX_CONNECTIONS` env var [#4673](https://github.com/appwrite/appwrite/pull/4673) +- Update Traefik 2.7 -> 2.9 [#4673](https://github.com/appwrite/appwrite/pull/4673) +- Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673) + # Version 1.1.0 ## Bugs From a59336ce952027f54876331aaaeb5a69417587f4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 21:28:49 +1300 Subject: [PATCH 23/25] Update compose template --- app/views/install/compose.phtml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 3ba1e8f124..0b83a12729 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -15,7 +15,7 @@ $image = $this->getParam('image', ''); services: traefik: - image: traefik:2.7 + image: traefik:2.9 container_name: appwrite-traefik <<: *x-logging command: @@ -30,6 +30,15 @@ services: ports: - :80 - :443 + ulimits: + nofile: + soft: 655350 + hard: 655350 + sysctls: + - net.core.somaxconn=1024 + - net.ipv4.tcp_rmem=1024 4096 16384 + - net.ipv4.tcp_wmem=1024 4096 16384 + - net.ipv4.ip_local_port_range=1025 65535 volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-config:/storage/config:ro @@ -93,6 +102,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_MAX_CONNECTIONS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -187,6 +197,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_MAX_CONNECTIONS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -611,7 +622,7 @@ services: - MYSQL_DATABASE=${_APP_DB_SCHEMA} - MYSQL_USER=${_APP_DB_USER} - MYSQL_PASSWORD=${_APP_DB_PASS} - command: 'mysqld --innodb-flush-method=fsync' + command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}' redis: image: redis:7.0.4-alpine From 6e0c54a1514d646e1fe90558870d558de806c40b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 21:59:23 +1300 Subject: [PATCH 24/25] Update pools version --- composer.json | 2 +- composer.lock | 21 +++++++-------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index e43a1c1688..36aed0d96b 100644 --- a/composer.json +++ b/composer.json @@ -57,7 +57,7 @@ "utopia-php/logger": "0.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.3.*", - "utopia-php/pools": "dev-feat-remove-dependencies as 0.4.0", + "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", "utopia-php/registry": "dev-feat-allow-params as 0.5.0", "utopia-php/storage": "0.11.*", diff --git a/composer.lock b/composer.lock index a3dfef5bac..f042279e4f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "41e3fe329598c2a3b4eb303f6605258d", + "content-hash": "b125039c64ae4cbe0d2a1b57322d0ebe", "packages": [ { "name": "adhocore/jwt", @@ -2255,16 +2255,16 @@ }, { "name": "utopia-php/pools", - "version": "dev-feat-remove-dependencies", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "bdc023b98023fc6a5978ee99cc750256acaeb194" + "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/bdc023b98023fc6a5978ee99cc750256acaeb194", - "reference": "bdc023b98023fc6a5978ee99cc750256acaeb194", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", + "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", "shasum": "" }, "require": { @@ -2300,9 +2300,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/feat-remove-dependencies" + "source": "https://github.com/utopia-php/pools/tree/0.4.1" }, - "time": "2022-11-13T22:14:50+00:00" + "time": "2022-11-15T08:55:16+00:00" }, { "name": "utopia-php/preloader", @@ -5216,12 +5216,6 @@ } ], "aliases": [ - { - "package": "utopia-php/pools", - "version": "dev-feat-remove-dependencies", - "alias": "0.4.0", - "alias_normalized": "0.4.0.0" - }, { "package": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5231,7 +5225,6 @@ ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/pools": 20, "utopia-php/registry": 20 }, "prefer-stable": false, From dc6c8d9b9ab01cddf265a93f043e8f7f409d51bc Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Nov 2022 22:16:25 +1300 Subject: [PATCH 25/25] Use 500 code for pool too small exception --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 7fd6acdcd1..af87960325 100644 --- a/app/init.php +++ b/app/init.php @@ -1137,7 +1137,7 @@ function getWorkerPoolSize(): int $workerConnections = $maxConnections - $reservedConnections; if ($workerCount > $workerConnections) { - throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.'); + throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); } return (int)($workerConnections / $workerCount);