Merge pull request #6949 from appwrite/feat-api-db-timeout

API DB timeout
2026-05-22 08:28:42 +00:00 · 2023-10-19 05:17:00 -04:00 · 2023-10-19 05:17:00 -04:00 · e2be9ea180
commit e2be9ea180
parent f80f7c8d9b a46b3d088c
10 changed files with 54766 additions and 42 deletions
--- a/app/config/errors.php
+++ b/app/config/errors.php
@ -460,12 +460,16 @@ return [
        'description' => 'Database not found',
        'code' => 404
    ],
-
    Exception::DATABASE_ALREADY_EXISTS => [
        'name' => Exception::DATABASE_ALREADY_EXISTS,
        'description' => 'Database already exists',
        'code' => 409
    ],
+    Exception::DATABASE_TIMEOUT => [
+        'name' => Exception::DATABASE_TIMEOUT,
+        'description' => 'Database timed out. Try adjusting your queries or adding an index.',
+        'code' => 408
+    ],

    /** Collections */
    Exception::COLLECTION_NOT_FOUND => [
--- a/app/controllers/api/databases.php
+++ b/app/controllers/api/databases.php
@ -12,6 +12,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Attributes;
 use Appwrite\Utopia\Database\Validator\Queries\Collections;
 use Appwrite\Utopia\Database\Validator\Queries\Databases;
 use Appwrite\Utopia\Database\Validator\Queries\Indexes;
+use Appwrite\Utopia\Request;
 use Appwrite\Utopia\Response;
 use MaxMind\Db\Reader;
 use Utopia\App;
@ -21,12 +22,13 @@ use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception\Authorization as AuthorizationException;
-use Utopia\Database\Exception\Conflict;
+use Utopia\Database\Exception\Conflict as ConflictException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Limit as LimitException;
+use Utopia\Database\Exception\Query as QueryException;
 use Utopia\Database\Exception\Restricted as RestrictedException;
 use Utopia\Database\Exception\Structure as StructureException;
-use Utopia\Database\Exception\Query as QueryException;
+use Utopia\Database\Exception\Timeout as TimeoutException;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
@ -71,7 +73,7 @@ use Utopia\Validator\WhiteList;
 * @throws RestrictedException
 * @throws StructureException
 * @throws \Utopia\Database\Exception
- * @throws Conflict
+ * @throws ConflictException
 * @throws Exception
 */
 function createAttribute(string $databaseId, string $collectionId, Document $attribute, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents): Document
@ -381,6 +383,18 @@ function updateAttribute(
    return $attribute;
 }

+App::init()
+    ->groups(['api', 'database'])
+    ->inject('request')
+    ->inject('dbForProject')
+    ->action(function (Request $request, Database $dbForProject) {
+        $timeout = \intval($request->getHeader('x-appwrite-timeout'));
+
+        if (!empty($timeout) && App::isDevelopment()) {
+            $dbForProject->setTimeout($timeout);
+        }
+    });
+
 App::post('/v1/databases')
    ->desc('Create database')
    ->groups(['api', 'database'])
@ -2500,8 +2514,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
            'orders' => $orders,
        ]);

-        $validator = new IndexValidator($dbForProject->getAdapter()->getMaxIndexLength());
-        if (!$validator->isValid($collection->setAttribute('indexes', $index, Document::SET_TYPE_APPEND))) {
+        $validator = new IndexValidator(
+            $collection->getAttribute('attributes'),
+            $dbForProject->getAdapter()->getMaxIndexLength()
+        );
+        if (!$validator->isValid($index)) {
            throw new Exception(Exception::INDEX_INVALID, $validator->getDescription());
        }

--- a/app/controllers/general.php
+++ b/app/controllers/general.php
@ -423,7 +423,7 @@ App::init()
            ->addHeader('Server', 'Appwrite')
            ->addHeader('X-Content-Type-Options', 'nosniff')
            ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE')
-            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma')
+            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma')
            ->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies')
            ->addHeader('Access-Control-Allow-Origin', $refDomain)
            ->addHeader('Access-Control-Allow-Credentials', 'true');
@ -587,7 +587,7 @@ App::options()
        $response
            ->addHeader('Server', 'Appwrite')
            ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE')
-            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies')
+            ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies')
            ->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies')
            ->addHeader('Access-Control-Allow-Origin', $origin)
            ->addHeader('Access-Control-Allow-Credentials', 'true')
@ -690,7 +690,11 @@ App::error()
                    break;
            }
        } elseif ($error instanceof Utopia\Database\Exception\Conflict) {
-            $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, null, null, $error);
+            $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error);
+            $code = $error->getCode();
+            $message = $error->getMessage();
+        } elseif ($error instanceof Utopia\Database\Exception\Timeout) {
+            $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error);
            $code = $error->getCode();
            $message = $error->getMessage();
        }
@ -706,6 +710,7 @@ App::error()
            case 402: // Error allowed publicly
            case 403: // Error allowed publicly
            case 404: // Error allowed publicly
+            case 408: // Error allowed publicly
            case 409: // Error allowed publicly
            case 412: // Error allowed publicly
            case 416: // Error allowed publicly
--- a/app/init.php
+++ b/app/init.php
@ -119,6 +119,7 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url';
 const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange';
 const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange';
 const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1073741824; // 2^32 bits / 4 bits per char
+const APP_DATABASE_TIMEOUT_MILLISECONDS = 15000;
 const APP_STORAGE_UPLOADS = '/storage/uploads';
 const APP_STORAGE_FUNCTIONS = '/storage/functions';
 const APP_STORAGE_BUILDS = '/storage/builds';
@ -1115,11 +1116,13 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
    $dbAdapter = $pools
        ->get($project->getAttribute('database'))
        ->pop()
-        ->getResource()
-    ;
+        ->getResource();

    $database = new Database($dbAdapter, $cache);
-    $database->setNamespace('_' . $project->getInternalId());
+
+    $database
+        ->setNamespace('_' . $project->getInternalId())
+        ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);

    return $database;
 }, ['pools', 'dbForConsole', 'cache', 'project']);
@ -1133,7 +1136,9 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) {

    $database = new Database($dbAdapter, $cache);

-    $database->setNamespace('_console');
+    $database
+        ->setNamespace('_console')
+        ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);

    return $database;
 }, ['pools', 'cache']);
@ -1150,7 +1155,11 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,

        if (isset($databases[$databaseName])) {
            $database = $databases[$databaseName];
-            $database->setNamespace('_' . $project->getInternalId());
+
+            $database
+                ->setNamespace('_' . $project->getInternalId())
+                ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
+
            return $database;
        }

@ -1163,7 +1172,9 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,

        $databases[$databaseName] = $database;

-        $database->setNamespace('_' . $project->getInternalId());
+        $database
+            ->setNamespace('_' . $project->getInternalId())
+            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);

        return $database;
    };
--- a/composer.json
+++ b/composer.json
@ -43,13 +43,13 @@
        "ext-sockets": "*",
        "appwrite/php-runtimes": "0.13.*",
        "appwrite/php-clamav": "2.0.*",
-        "utopia-php/abuse": "0.31.*",
+        "utopia-php/abuse": "0.32.*",
        "utopia-php/analytics": "0.10.*",
-        "utopia-php/audit": "0.33.*",
+        "utopia-php/audit": "0.34.*",
        "utopia-php/cache": "0.8.*",
        "utopia-php/cli": "0.15.*",
        "utopia-php/config": "0.2.*",
-        "utopia-php/database": "0.43.*",
+        "utopia-php/database": "0.44.*",
        "utopia-php/domains": "0.3.*",
        "utopia-php/dsn": "0.1.*",
        "utopia-php/framework": "0.31.0",
--- a/composer.lock
+++ b/composer.lock
@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "3a0624bf1df70e602233efa5916aa6ce",
+    "content-hash": "6ff937a260c3e0c09de9eb5e073d830d",
    "packages": [
        {
            "name": "adhocore/jwt",
@ -1681,23 +1681,23 @@
        },
        {
            "name": "utopia-php/abuse",
-            "version": "0.31.1",
+            "version": "0.32.0",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/abuse.git",
-                "reference": "b2ad372d1070f55f9545cb811b6ed2d40094e6dd"
+                "reference": "9717ffb2d7711f3fd621bb6df3edf5724c08ea78"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/abuse/zipball/b2ad372d1070f55f9545cb811b6ed2d40094e6dd",
-                "reference": "b2ad372d1070f55f9545cb811b6ed2d40094e6dd",
+                "url": "https://api.github.com/repos/utopia-php/abuse/zipball/9717ffb2d7711f3fd621bb6df3edf5724c08ea78",
+                "reference": "9717ffb2d7711f3fd621bb6df3edf5724c08ea78",
                "shasum": ""
            },
            "require": {
                "ext-curl": "*",
                "ext-pdo": "*",
                "php": ">=8.0",
-                "utopia-php/database": "0.43.*"
+                "utopia-php/database": "0.44.*"
            },
            "require-dev": {
                "laravel/pint": "1.5.*",
@ -1724,9 +1724,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/abuse/issues",
-                "source": "https://github.com/utopia-php/abuse/tree/0.31.1"
+                "source": "https://github.com/utopia-php/abuse/tree/0.32.0"
            },
-            "time": "2023-08-29T11:07:46+00:00"
+            "time": "2023-10-18T07:28:55+00:00"
        },
        {
            "name": "utopia-php/analytics",
@ -1776,21 +1776,21 @@
        },
        {
            "name": "utopia-php/audit",
-            "version": "0.33.1",
+            "version": "0.34.0",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/audit.git",
-                "reference": "c117e8e9ce4e3e1b369e8b5b55b2d6ab3138eadd"
+                "reference": "cf34cc3f9f20da4e574a9be4517e1a11025a858f"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/audit/zipball/c117e8e9ce4e3e1b369e8b5b55b2d6ab3138eadd",
-                "reference": "c117e8e9ce4e3e1b369e8b5b55b2d6ab3138eadd",
+                "url": "https://api.github.com/repos/utopia-php/audit/zipball/cf34cc3f9f20da4e574a9be4517e1a11025a858f",
+                "reference": "cf34cc3f9f20da4e574a9be4517e1a11025a858f",
                "shasum": ""
            },
            "require": {
                "php": ">=8.0",
-                "utopia-php/database": "0.43.*"
+                "utopia-php/database": "0.44.*"
            },
            "require-dev": {
                "laravel/pint": "1.5.*",
@ -1817,9 +1817,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/audit/issues",
-                "source": "https://github.com/utopia-php/audit/tree/0.33.1"
+                "source": "https://github.com/utopia-php/audit/tree/0.34.0"
            },
-            "time": "2023-08-29T11:07:40+00:00"
+            "time": "2023-10-18T07:43:25+00:00"
        },
        {
            "name": "utopia-php/cache",
@ -1972,16 +1972,16 @@
        },
        {
            "name": "utopia-php/database",
-            "version": "0.43.5",
+            "version": "0.44.2",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/database.git",
-                "reference": "5f7b05189cfbcc0506090498c580c5765375a00a"
+                "reference": "591cadbc2c622a3304aae9a16ebfdbe75ef33a06"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/5f7b05189cfbcc0506090498c580c5765375a00a",
-                "reference": "5f7b05189cfbcc0506090498c580c5765375a00a",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/591cadbc2c622a3304aae9a16ebfdbe75ef33a06",
+                "reference": "591cadbc2c622a3304aae9a16ebfdbe75ef33a06",
                "shasum": ""
            },
            "require": {
@ -2022,9 +2022,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.43.5"
+                "source": "https://github.com/utopia-php/database/tree/0.44.2"
            },
-            "time": "2023-10-06T06:49:47+00:00"
+            "time": "2023-10-19T07:39:00+00:00"
        },
        {
            "name": "utopia-php/domains",
--- a/src/Appwrite/Extend/Exception.php
+++ b/src/Appwrite/Extend/Exception.php
@ -145,6 +145,7 @@ class Exception extends \Exception
    /** Databases */
    public const DATABASE_NOT_FOUND                = 'database_not_found';
    public const DATABASE_ALREADY_EXISTS           = 'database_already_exists';
+    public const DATABASE_TIMEOUT                  = 'database_timeout';

    /** Collections */
    public const COLLECTION_NOT_FOUND              = 'collection_not_found';
--- a/tests/e2e/General/HTTPTest.php
+++ b/tests/e2e/General/HTTPTest.php
@ -31,7 +31,7 @@ class HTTPTest extends Scope
        $this->assertEquals(204, $response['headers']['status-code']);
        $this->assertEquals('Appwrite', $response['headers']['server']);
        $this->assertEquals('GET, POST, PUT, PATCH, DELETE', $response['headers']['access-control-allow-methods']);
-        $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']);
+        $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']);
        $this->assertEquals('X-Fallback-Cookies', $response['headers']['access-control-expose-headers']);
        $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
        $this->assertEquals('true', $response['headers']['access-control-allow-credentials']);
--- a/tests/e2e/Services/Databases/DatabasesBase.php
+++ b/tests/e2e/Services/Databases/DatabasesBase.php
@ -5,12 +5,10 @@ namespace Tests\E2E\Services\Databases;
 use Appwrite\Extend\Exception;
 use Tests\E2E\Client;
 use Utopia\Database\Database;
-use Utopia\Database\Document;
 use Utopia\Database\DateTime;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
-use Utopia\Database\Query;
 use Utopia\Database\Validator\Datetime as DatetimeValidator;

 trait DatabasesBase
@ -4216,4 +4214,70 @@ trait DatabasesBase

        $this->assertEquals(200, $update['headers']['status-code']);
    }
+
+    /**
+     * @depends testCreateDatabase
+     */
+    public function testTimeout(array $data): void
+    {
+        $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $data['databaseId'] . '/collections', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]), [
+            'collectionId' => ID::unique(),
+            'name' => 'Slow Queries',
+            'documentSecurity' => true,
+            'permissions' => [
+                Permission::create(Role::user($this->getUser()['$id'])),
+            ],
+        ]);
+
+        $this->assertEquals(201, $collection['headers']['status-code']);
+
+        $data = [
+            '$id' => $collection['body']['$id'],
+            'databaseId' => $collection['body']['databaseId']
+        ];
+
+        $longtext = $this->client->call(Client::METHOD_POST, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/attributes/string', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]), [
+            'key' => 'longtext',
+            'size' => 100000000,
+            'required' => false,
+            'default' => null,
+        ]);
+
+        $this->assertEquals($longtext['headers']['status-code'], 202);
+
+        for ($i = 0; $i < 1; $i++) {
+            $this->client->call(Client::METHOD_POST, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()), [
+                'documentId' => ID::unique(),
+                'data' => [
+                    'longtext' => file_get_contents('tests/resources/longtext.txt'),
+                ],
+                'permissions' => [
+                    Permission::read(Role::user($this->getUser()['$id'])),
+                    Permission::update(Role::user($this->getUser()['$id'])),
+                    Permission::delete(Role::user($this->getUser()['$id'])),
+                ]
+            ]);
+        }
+
+        $response = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-timeout' => 1,
+        ], $this->getHeaders()), [
+            'queries' => ['notEqual("longtext", "appwrite")'],
+        ]);
+
+        $this->assertEquals(408, $response['headers']['status-code']);
+    }
 }
--- a/tests/resources/longtext.txt
+++ b/tests/resources/longtext.txt