Merge pull request #9507 from appwrite/feat-improve-delete-queries

Feat improve delete queries

app/cli.php

@@ -25,7 +25,7 @@ use Utopia\Queue\Publisher;
 use Utopia\Registry\Registry;
 use Utopia\System\System;
 
-// overwriting runtimes to be architectur agnostic for CLI
+// Overwriting runtimes to be architecture agnostic for CLI
 Config::setParam('runtimes', (new Runtimes('v4'))->getAll(supported: false));
 
 // require controllers after overwriting runtimes
@@ -43,8 +43,7 @@ CLI::setResource('cache', function ($pools) {
         $adapters[] = $pools
             ->get($value)
             ->pop()
-            ->getResource()
-        ;
+            ->getResource();
     }
 
     return new Cache(new Sharding($adapters));
@@ -187,7 +186,7 @@ CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) {
         $database
             ->setSharedTables(true)
             ->setNamespace('logsV1')
-            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
+            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_TASK)
             ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
 
         // set tenant

app/init.php

@@ -134,7 +134,9 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url';
 const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange';
 const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange';
 const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 bits per char
-const APP_DATABASE_TIMEOUT_MILLISECONDS = 15_000;
+const APP_DATABASE_TIMEOUT_MILLISECONDS_API = 15 * 1000; // 15 seconds
+const APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER = 300 * 1000; // 5 minutes
+const APP_DATABASE_TIMEOUT_MILLISECONDS_TASK = 300 * 1000; // 5 minutes
 const APP_DATABASE_QUERY_MAX_VALUES = 500;
 const APP_STORAGE_UPLOADS = '/storage/uploads';
 const APP_STORAGE_FUNCTIONS = '/storage/functions';
@@ -1434,7 +1436,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform
     $database
         ->setMetadata('host', \gethostname())
         ->setMetadata('project', $project->getId())
-        ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
+        ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
         ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
 
     $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
@@ -1466,7 +1468,7 @@ App::setResource('dbForPlatform', function (Group $pools, Cache $cache) {
         ->setNamespace('_console')
         ->setMetadata('host', \gethostname())
         ->setMetadata('project', 'console')
-        ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
+        ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
         ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
 
     return $database;
@@ -1491,7 +1493,7 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform
             $database
                 ->setMetadata('host', \gethostname())
                 ->setMetadata('project', $project->getId())
-                ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
+                ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
                 ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
 
             $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
@@ -1549,7 +1551,7 @@ App::setResource('getLogsDB', function (Group $pools, Cache $cache) {
         $database
             ->setSharedTables(true)
             ->setNamespace('logsV1')
-            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
+            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
             ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
 
         // set tenant

app/worker.php

@@ -112,6 +112,8 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
             ->setNamespace('_' . $project->getInternalId());
     }
 
+    $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER);
+
     return $database;
 }, ['cache', 'register', 'message', 'project', 'dbForPlatform']);
 
@@ -173,6 +175,8 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf
                 ->setNamespace('_' . $project->getInternalId());
         }
 
+        $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER);
+
         return $database;
     };
 }, ['pools', 'dbForPlatform', 'cache']);
@@ -198,7 +202,7 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) {
         $database
             ->setSharedTables(true)
             ->setNamespace('logsV1')
-            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
+            ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER)
             ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
 
         // set tenant

composer.json

@@ -45,13 +45,13 @@
         "ext-sockets": "*",
         "appwrite/php-runtimes": "0.16.*",
         "appwrite/php-clamav": "2.0.*",
-        "utopia-php/abuse": "0.51.*",
+        "utopia-php/abuse": "0.52.*",
         "utopia-php/analytics": "0.10.*",
-        "utopia-php/audit": "0.54.0",
+        "utopia-php/audit": "0.55.*",
         "utopia-php/cache": "0.12.*",
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
-        "utopia-php/database": "0.60.*",
+        "utopia-php/database": "0.61.*",
         "utopia-php/domains": "0.5.*",
         "utopia-php/dsn": "0.2.1",
         "utopia-php/framework": "0.33.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "418a223d14b5ec5a5a8751623d5172b6",
+    "content-hash": "44c6436ced36b0b026139edba252052e",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -709,16 +709,16 @@
         },
         {
             "name": "google/protobuf",
-            "version": "v4.30.0",
+            "version": "v4.30.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/protocolbuffers/protobuf-php.git",
-                "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6"
+                "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6",
-                "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6",
+                "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24",
+                "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24",
                 "shasum": ""
             },
             "require": {
@@ -747,9 +747,9 @@
                 "proto"
             ],
             "support": {
-                "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0"
+                "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1"
             },
-            "time": "2025-03-04T22:54:49+00:00"
+            "time": "2025-03-13T21:08:17+00:00"
         },
         {
             "name": "jean85/pretty-package-versions",
@@ -3364,16 +3364,16 @@
         },
         {
             "name": "utopia-php/abuse",
-            "version": "0.51.0",
+            "version": "0.52.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/abuse.git",
-                "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e"
+                "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/abuse/zipball/661687b03277f1d202a0e8cf9da6e58c97da2b5e",
-                "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e",
+                "url": "https://api.github.com/repos/utopia-php/abuse/zipball/a0d6421e7e5baa3ac02755496dca9fdeaa814b93",
+                "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93",
                 "shasum": ""
             },
             "require": {
@@ -3381,7 +3381,7 @@
                 "ext-pdo": "*",
                 "ext-redis": "*",
                 "php": ">=8.0",
-                "utopia-php/database": "0.60.*"
+                "utopia-php/database": "0.*.*"
             },
             "require-dev": {
                 "laravel/pint": "1.*",
@@ -3409,9 +3409,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/abuse/issues",
-                "source": "https://github.com/utopia-php/abuse/tree/0.51.0"
+                "source": "https://github.com/utopia-php/abuse/tree/0.52.0"
             },
-            "time": "2025-02-17T11:10:18+00:00"
+            "time": "2025-03-06T03:48:29+00:00"
         },
         {
             "name": "utopia-php/analytics",
@@ -3461,21 +3461,21 @@
         },
         {
             "name": "utopia-php/audit",
-            "version": "0.54.0",
+            "version": "0.55.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/audit.git",
-                "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975"
+                "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/audit/zipball/1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975",
-                "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975",
+                "url": "https://api.github.com/repos/utopia-php/audit/zipball/9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518",
+                "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518",
                 "shasum": ""
             },
             "require": {
                 "php": ">=8.0",
-                "utopia-php/database": "0.60.*"
+                "utopia-php/database": "0.*.*"
             },
             "require-dev": {
                 "laravel/pint": "1.*",
@@ -3502,9 +3502,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/audit/issues",
-                "source": "https://github.com/utopia-php/audit/tree/0.54.0"
+                "source": "https://github.com/utopia-php/audit/tree/0.55.0"
             },
-            "time": "2025-02-25T07:21:07+00:00"
+            "time": "2025-03-06T03:47:47+00:00"
         },
         {
             "name": "utopia-php/cache",
@@ -3705,16 +3705,16 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.60.6",
+            "version": "0.61.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "f3c9aa964b39c6205069f038a26e709a15541406"
+                "reference": "2e0165bd14a570ec151f400ed381108e81d15b94"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/f3c9aa964b39c6205069f038a26e709a15541406",
-                "reference": "f3c9aa964b39c6205069f038a26e709a15541406",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94",
+                "reference": "2e0165bd14a570ec151f400ed381108e81d15b94",
                 "shasum": ""
             },
             "require": {
@@ -3755,9 +3755,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.60.6"
+                "source": "https://github.com/utopia-php/database/tree/0.61.1"
             },
-            "time": "2025-03-05T01:23:14+00:00"
+            "time": "2025-03-14T01:19:38+00:00"
         },
         {
             "name": "utopia-php/domains",
@@ -4159,16 +4159,16 @@
         },
         {
             "name": "utopia-php/migration",
-            "version": "0.6.20",
+            "version": "0.6.22",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/migration.git",
-                "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba"
+                "reference": "a0269746bd318ff0993f5aa008675b971689d5b5"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/migration/zipball/8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba",
-                "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba",
+                "url": "https://api.github.com/repos/utopia-php/migration/zipball/a0269746bd318ff0993f5aa008675b971689d5b5",
+                "reference": "a0269746bd318ff0993f5aa008675b971689d5b5",
                 "shasum": ""
             },
             "require": {
@@ -4176,7 +4176,7 @@
                 "ext-curl": "*",
                 "ext-openssl": "*",
                 "php": ">=8.1",
-                "utopia-php/database": "0.60.*",
+                "utopia-php/database": "0.61.*",
                 "utopia-php/dsn": "0.2.*",
                 "utopia-php/framework": "0.33.*",
                 "utopia-php/storage": "0.18.*"
@@ -4209,9 +4209,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/migration/issues",
-                "source": "https://github.com/utopia-php/migration/tree/0.6.20"
+                "source": "https://github.com/utopia-php/migration/tree/0.6.22"
             },
-            "time": "2025-02-17T11:02:15+00:00"
+            "time": "2025-03-13T07:35:55+00:00"
         },
         {
             "name": "utopia-php/mongo",
@@ -4810,16 +4810,16 @@
         },
         {
             "name": "utopia-php/vcs",
-            "version": "0.9.3",
+            "version": "0.9.4",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/vcs.git",
-                "reference": "865a00c67e81a20938b883f9aa802303790dd3b5"
+                "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/865a00c67e81a20938b883f9aa802303790dd3b5",
-                "reference": "865a00c67e81a20938b883f9aa802303790dd3b5",
+                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/1a8d280b176acc99ea8d9e7364b8767cbb206b4a",
+                "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a",
                 "shasum": ""
             },
             "require": {
@@ -4854,9 +4854,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/vcs/issues",
-                "source": "https://github.com/utopia-php/vcs/tree/0.9.3"
+                "source": "https://github.com/utopia-php/vcs/tree/0.9.4"
             },
-            "time": "2025-02-26T16:33:35+00:00"
+            "time": "2025-03-13T10:09:45+00:00"
         },
         {
             "name": "utopia-php/websocket",
@@ -8402,7 +8402,7 @@
     ],
     "aliases": [],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {},
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": {

src/Appwrite/Platform/Workers/Deletes.php

@@ -387,6 +387,7 @@ class Deletes extends Action
 
         $query = [
             Query::lessThan('accessedAt', $datetime),
+            Query::orderDesc('accessedAt')
         ];
 
         $this->deleteByGroup(
@@ -420,6 +421,7 @@ class Deletes extends Action
         // Delete Usage stats from projectDB
         $this->deleteByGroup('stats', [
             Query::lessThan('time', $hourlyUsageRetentionDatetime),
+            Query::orderDesc('time'),
             Query::equal('period', ['1h']),
         ], $dbForProject);
 
@@ -430,6 +432,7 @@ class Deletes extends Action
             // Delete Usage stats from logsDB
             $this->deleteByGroup('stats', [
                 Query::lessThan('time', $hourlyUsageRetentionDatetime),
+                Query::orderDesc('time'),
                 Query::equal('period', ['1h']),
             ], $dbForLogs);
         }
@@ -688,9 +691,11 @@ class Deletes extends Action
     private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
     {
         $dbForProject = $getProjectDB($project);
+
         // Delete Executions
         $this->deleteByGroup('executions', [
-            Query::lessThan('$createdAt', $datetime)
+            Query::lessThan('$createdAt', $datetime),
+            Query::orderDesc('$createdAt'),
         ], $dbForProject);
     }
 
@@ -708,7 +713,8 @@ class Deletes extends Action
 
         // Delete Sessions
         $this->deleteByGroup('sessions', [
-            Query::lessThan('$createdAt', $expired)
+            Query::lessThan('$createdAt', $expired),
+            Query::orderDesc('$createdAt'),
         ], $dbForProject);
     }
 
@@ -722,7 +728,8 @@ class Deletes extends Action
     {
         // Delete Dead Realtime Logs
         $this->deleteByGroup('realtime', [
-            Query::lessThan('timestamp', $datetime)
+            Query::lessThan('timestamp', $datetime),
+            Query::orderDesc('timestamp'),
         ], $dbForPlatform);
     }