Merge pull request #9190 from appwrite/fix-dead-connections

Fix dead connections
2026-05-14 12:38:31 +00:00 · 2025-01-08 15:00:15 +05:30 · 2025-01-08 15:00:15 +05:30 · 910bd4d8ea
commit 910bd4d8ea
parent f4af241804 941c9f1905
6 changed files with 98 additions and 48 deletions
--- a/app/controllers/general.php
+++ b/app/controllers/general.php
@ -21,6 +21,7 @@ use Appwrite\Utopia\Response\Filters\V18 as ResponseV18;
 use Appwrite\Utopia\View;
 use Executor\Executor;
 use MaxMind\Db\Reader;
+use Swoole\Database\DetectsLostConnections;
 use Swoole\Http\Request as SwooleRequest;
 use Utopia\App;
 use Utopia\CLI\Console;
@ -28,6 +29,7 @@ use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
+use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
@ -38,6 +40,7 @@ use Utopia\Logger\Adapter\Sentry;
 use Utopia\Logger\Log;
 use Utopia\Logger\Log\User;
 use Utopia\Logger\Logger;
+use Utopia\Pools\Connection;
 use Utopia\System\System;
 use Utopia\Validator\Hostname;
 use Utopia\Validator\Text;
@ -746,7 +749,16 @@ App::error()
    ->inject('logger')
    ->inject('log')
    ->inject('queueForUsage')
-    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) {
+    ->inject('connectionForProject')
+    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage, Connection $connectionForProject) {
+        if (
+            ($error instanceof PDOException || $error instanceof DatabaseException)
+            && DetectsLostConnections::causedByLostConnection($error)
+        ) {
+            // Mark connection as unhealthy so it will be recycled on next reclaim.
+            $connectionForProject->setHealthy(false);
+        }
+
        $version = System::getEnv('_APP_VERSION', 'UNKNOWN');
        $route = $utopia->getRoute();
        $class = \get_class($error);
--- a/app/http.php
+++ b/app/http.php
@ -5,6 +5,7 @@ require_once __DIR__ . '/../vendor/autoload.php';
 use Appwrite\Utopia\Request;
 use Appwrite\Utopia\Response;
 use Swoole\Constant;
+use Swoole\Database\DetectsLostConnections;
 use Swoole\Http\Request as SwooleRequest;
 use Swoole\Http\Response as SwooleResponse;
 use Swoole\Http\Server;
@ -17,6 +18,7 @@ use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
+use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Duplicate;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
@ -346,6 +348,15 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool

        $app->run($request, $response);
    } catch (\Throwable $th) {
+        if (
+            ($th instanceof PDOException || $th instanceof DatabaseException)
+            && DetectsLostConnections::causedByLostConnection($th)
+        ) {
+            // Mark connection as unhealthy so it will be recycled on next reclaim.
+            $connectionForProject = $app->getResource('connectionForProject');
+            $connectionForProject->setHealthy(false);
+        }
+
        $version = System::getEnv('_APP_VERSION', 'UNKNOWN');

        $logger = $app->getResource("logger");
--- a/app/init.php
+++ b/app/init.php
@ -74,6 +74,7 @@ use Utopia\Logger\Adapter\Raygun;
 use Utopia\Logger\Adapter\Sentry;
 use Utopia\Logger\Log;
 use Utopia\Logger\Logger;
+use Utopia\Pools\Connection as PoolConnection;
 use Utopia\Pools\Group;
 use Utopia\Pools\Pool;
 use Utopia\Queue;
@ -1412,7 +1413,26 @@ App::setResource('console', function () {
    ]);
 }, []);

-App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) {
+App::setResource('connectionForProject', function (Group $pools, Document $project) {
+    if ($project->isEmpty() || $project->getId() === 'console') {
+        return $pools
+            ->get('console')
+            ->pop();
+    }
+
+    try {
+        $dsn = new DSN($project->getAttribute('database'));
+    } catch (\InvalidArgumentException) {
+        // TODO: Temporary until all projects are using shared tables
+        $dsn = new DSN('mysql://' . $project->getAttribute('database'));
+    }
+
+    return $pools
+        ->get($dsn->getHost())
+        ->pop();
+}, ['pools', 'project']);
+
+App::setResource('dbForProject', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, Cache $cache, Document $project) {
    if ($project->isEmpty() || $project->getId() === 'console') {
        return $dbForPlatform;
    }
@ -1424,12 +1444,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform
        $dsn = new DSN('mysql://' . $project->getAttribute('database'));
    }

-    $dbAdapter = $pools
-        ->get($dsn->getHost())
-        ->pop()
-        ->getResource();
-
-    $database = new Database($dbAdapter, $cache);
+    $database = new Database($connectionForProject->getResource(), $cache);

    $database
        ->setMetadata('host', \gethostname())
@ -1452,7 +1467,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform
    }

    return $database;
-}, ['pools', 'dbForPlatform', 'cache', 'project']);
+}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache', 'project']);

 App::setResource('dbForPlatform', function (Group $pools, Cache $cache) {
    $dbAdapter = $pools
--- a/app/worker.php
+++ b/app/worker.php
@ -16,6 +16,7 @@ use Appwrite\Event\Migration;
 use Appwrite\Event\Usage;
 use Appwrite\Event\UsageDump;
 use Appwrite\Platform\Appwrite;
+use Swoole\Database\DetectsLostConnections;
 use Swoole\Runtime;
 use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
 use Utopia\Cache\Adapter\Sharding;
@ -25,11 +26,13 @@ use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
+use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Validator\Authorization;
 use Utopia\DSN\DSN;
 use Utopia\Logger\Log;
 use Utopia\Logger\Logger;
 use Utopia\Platform\Service;
+use Utopia\Pools\Connection as PoolConnection;
 use Utopia\Pools\Group;
 use Utopia\Queue\Connection;
 use Utopia\Queue\Message;
@ -66,13 +69,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo
    return $dbForPlatform->getDocument('projects', $project->getId());
 }, ['message', 'dbForPlatform']);

-Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) {
+Server::setResource('connectionForProject', function (Group $pools, Document $project) {
    if ($project->isEmpty() || $project->getId() === 'console') {
-        return $dbForPlatform;
+        return $pools
+            ->get('console')
+            ->pop();
    }

-    $pools = $register->get('pools');
-
    try {
        $dsn = new DSN($project->getAttribute('database'));
    } catch (\InvalidArgumentException) {
@ -80,12 +83,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
        $dsn = new DSN('mysql://' . $project->getAttribute('database'));
    }

-    $adapter = $pools
+    return $pools
        ->get($dsn->getHost())
-        ->pop()
-        ->getResource();
+        ->pop();
+}, ['pools', 'project']);

-    $database = new Database($adapter, $cache);
+Server::setResource('dbForProject', function (PoolConnection $connectionForProject, Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) {
+    if ($project->isEmpty() || $project->getId() === 'console') {
+        return $dbForPlatform;
+    }
+
+    $database = new Database($connectionForProject->getResource(), $cache);

    try {
        $dsn = new DSN($project->getAttribute('database'));
@ -109,12 +117,12 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
    }

    return $database;
-}, ['cache', 'register', 'message', 'project', 'dbForPlatform']);
+}, ['connectionForProject', 'cache', 'register', 'message', 'project', 'dbForPlatform']);

-Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) {
+Server::setResource('getProjectDB', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, $cache) {
    $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools

-    return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database {
+    return function (Document $project) use ($pools, $connectionForProject, $dbForPlatform, $cache, &$databases): Database {
        if ($project->isEmpty() || $project->getId() === 'console') {
            return $dbForPlatform;
        }
@ -146,12 +154,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf
            return $database;
        }

-        $dbAdapter = $pools
-            ->get($dsn->getHost())
-            ->pop()
-            ->getResource();
-
-        $database = new Database($dbAdapter, $cache);
+        $database = new Database($connectionForProject->getResource(), $cache);

        $databases[$dsn->getHost()] = $database;

@ -171,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf

        return $database;
    };
-}, ['pools', 'dbForPlatform', 'cache']);
+}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache']);

 Server::setResource('abuseRetention', function () {
    return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400);
@ -410,7 +413,16 @@ $worker
    ->inject('log')
    ->inject('pools')
    ->inject('project')
-    ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project) use ($queueName) {
+    ->inject('connectionForProject')
+    ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, PoolConnection $connectionForProject) use ($queueName) {
+        if (
+            ($error instanceof PDOException || $error instanceof DatabaseException)
+            && DetectsLostConnections::causedByLostConnection($error)
+        ) {
+            // Mark connection as unhealthy, it will be recycled on next reclaim.
+            $connectionForProject->setHealthy(false);
+        }
+
        $pools->reclaim();
        $version = System::getEnv('_APP_VERSION', 'UNKNOWN');

--- a/composer.json
+++ b/composer.json
@ -63,7 +63,7 @@
        "utopia-php/migration": "0.6.*",
        "utopia-php/orchestration": "0.9.*",
        "utopia-php/platform": "0.7.*",
-        "utopia-php/pools": "0.5.*",
+        "utopia-php/pools": "0.6.*",
        "utopia-php/preloader": "0.2.*",
        "utopia-php/queue": "0.7.*",
        "utopia-php/registry": "0.5.*",
--- a/composer.lock
+++ b/composer.lock
@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "6b136b5490c0d5d331eac0d70bb3e198",
+    "content-hash": "7b5b5926b452186543903eb539f59c2d",
    "packages": [
        {
            "name": "adhocore/jwt",
@ -3929,16 +3929,16 @@
        },
        {
            "name": "utopia-php/migration",
-            "version": "0.6.13",
+            "version": "0.6.14",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/migration.git",
-                "reference": "68d9b0a9477755afcda607e7e8109785cae17a13"
+                "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13",
-                "reference": "68d9b0a9477755afcda607e7e8109785cae17a13",
+                "url": "https://api.github.com/repos/utopia-php/migration/zipball/59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2",
+                "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2",
                "shasum": ""
            },
            "require": {
@ -3979,9 +3979,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/migration/issues",
-                "source": "https://github.com/utopia-php/migration/tree/0.6.13"
+                "source": "https://github.com/utopia-php/migration/tree/0.6.14"
            },
-            "time": "2024-11-26T13:57:53+00:00"
+            "time": "2025-01-08T01:07:25+00:00"
        },
        {
            "name": "utopia-php/mongo",
@ -4145,25 +4145,25 @@
        },
        {
            "name": "utopia-php/pools",
-            "version": "0.5.0",
+            "version": "0.6.0",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/pools.git",
-                "reference": "6f716a213a08db95eda1b5dddfa90983c1834817"
+                "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817",
-                "reference": "6f716a213a08db95eda1b5dddfa90983c1834817",
+                "url": "https://api.github.com/repos/utopia-php/pools/zipball/59414ab7b57728edfde6d5ccc5a2583b7967ac18",
+                "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18",
                "shasum": ""
            },
            "require": {
-                "php": ">=8.0"
+                "php": ">=8.3"
            },
            "require-dev": {
-                "laravel/pint": "1.2.*",
-                "phpstan/phpstan": "1.8.*",
-                "phpunit/phpunit": "^9.3"
+                "laravel/pint": "1.*",
+                "phpstan/phpstan": "1.*",
+                "phpunit/phpunit": "11.*"
            },
            "type": "library",
            "autoload": {
@ -4190,9 +4190,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/pools/issues",
-                "source": "https://github.com/utopia-php/pools/tree/0.5.0"
+                "source": "https://github.com/utopia-php/pools/tree/0.6.0"
            },
-            "time": "2024-04-19T11:11:54+00:00"
+            "time": "2025-01-08T07:58:42+00:00"
        },
        {
            "name": "utopia-php/preloader",
@ -8556,7 +8556,7 @@
    ],
    "aliases": [],
    "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {},
    "prefer-stable": false,
    "prefer-lowest": false,
    "platform": {
@ -8580,5 +8580,5 @@
    "platform-overrides": {
        "php": "8.3"
    },
-    "plugin-api-version": "2.2.0"
+    "plugin-api-version": "2.6.0"
 }