Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-24 09:28:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #9507 from appwrite/feat-improve-delete-queries

Browse source 
Feat improve delete queries
This commit is contained in:
Jake Barnby 2025-03-14 18:37:56 +13:00 committed by GitHub
commit 810d650aba
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 69 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
app/cli.php
View file

@ -25,7 +25,7 @@ use Utopia\Queue\Publisher;
use Utopia\Registry\Registry; use Utopia\Registry\Registry;
use Utopia\System\System; use Utopia\System\System;
// overwriting runtimes to be architectur agnostic for CLI // Overwriting runtimes to be architecture agnostic for CLI
Config::setParam('runtimes', (new Runtimes('v4'))->getAll(supported: false)); Config::setParam('runtimes', (new Runtimes('v4'))->getAll(supported: false));
// require controllers after overwriting runtimes // require controllers after overwriting runtimes
@ -43,8 +43,7 @@ CLI::setResource('cache', function ($pools) {
$adapters[] = $pools $adapters[] = $pools
->get($value) ->get($value)
->pop() ->pop()
->getResource() ->getResource();
;
} }
return new Cache(new Sharding($adapters)); return new Cache(new Sharding($adapters));
@ -187,7 +186,7 @@ CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) {
$database $database
->setSharedTables(true) ->setSharedTables(true)
->setNamespace('logsV1') ->setNamespace('logsV1')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_TASK)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
// set tenant // set tenant

12
app/init.php
View file

@ -134,7 +134,9 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url';
const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange'; const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange';
const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange'; const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange';
const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 bits per char const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 bits per char
const APP_DATABASE_TIMEOUT_MILLISECONDS = 15_000; const APP_DATABASE_TIMEOUT_MILLISECONDS_API = 15 * 1000; // 15 seconds
const APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER = 300 * 1000; // 5 minutes
const APP_DATABASE_TIMEOUT_MILLISECONDS_TASK = 300 * 1000; // 5 minutes
const APP_DATABASE_QUERY_MAX_VALUES = 500; const APP_DATABASE_QUERY_MAX_VALUES = 500;
const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_UPLOADS = '/storage/uploads';
const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_FUNCTIONS = '/storage/functions';
@ -1434,7 +1436,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform
$database $database
->setMetadata('host', \gethostname()) ->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId()) ->setMetadata('project', $project->getId())
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
@ -1466,7 +1468,7 @@ App::setResource('dbForPlatform', function (Group $pools, Cache $cache) {
->setNamespace('_console') ->setNamespace('_console')
->setMetadata('host', \gethostname()) ->setMetadata('host', \gethostname())
->setMetadata('project', 'console') ->setMetadata('project', 'console')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
return $database; return $database;
@ -1491,7 +1493,7 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform
$database $database
->setMetadata('host', \gethostname()) ->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId()) ->setMetadata('project', $project->getId())
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
@ -1549,7 +1551,7 @@ App::setResource('getLogsDB', function (Group $pools, Cache $cache) {
$database $database
->setSharedTables(true) ->setSharedTables(true)
->setNamespace('logsV1') ->setNamespace('logsV1')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
// set tenant // set tenant

6
app/worker.php
View file

@ -112,6 +112,8 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
->setNamespace('_' . $project->getInternalId()); ->setNamespace('_' . $project->getInternalId());
} }
$database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER);
return $database; return $database;
}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); }, ['cache', 'register', 'message', 'project', 'dbForPlatform']);
@ -173,6 +175,8 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf
->setNamespace('_' . $project->getInternalId()); ->setNamespace('_' . $project->getInternalId());
} }
$database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER);
return $database; return $database;
}; };
}, ['pools', 'dbForPlatform', 'cache']); }, ['pools', 'dbForPlatform', 'cache']);
@ -198,7 +202,7 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) {
$database $database
->setSharedTables(true) ->setSharedTables(true)
->setNamespace('logsV1') ->setNamespace('logsV1')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
// set tenant // set tenant

6
composer.json
View file

@ -45,13 +45,13 @@
"ext-sockets": "*", "ext-sockets": "*",
"appwrite/php-runtimes": "0.16.*", "appwrite/php-runtimes": "0.16.*",
"appwrite/php-clamav": "2.0.*", "appwrite/php-clamav": "2.0.*",
"utopia-php/abuse": "0.51.*", "utopia-php/abuse": "0.52.*",
"utopia-php/analytics": "0.10.*", "utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.54.0", "utopia-php/audit": "0.55.*",
"utopia-php/cache": "0.12.*", "utopia-php/cache": "0.12.*",
"utopia-php/cli": "0.15.*", "utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*", "utopia-php/config": "0.2.*",
"utopia-php/database": "0.60.*", "utopia-php/database": "0.61.*",
"utopia-php/domains": "0.5.*", "utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.1", "utopia-php/dsn": "0.2.1",
"utopia-php/framework": "0.33.*", "utopia-php/framework": "0.33.*",

82
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "418a223d14b5ec5a5a8751623d5172b6", "content-hash": "44c6436ced36b0b026139edba252052e",
"packages": [ "packages": [
{ {
"name": "adhocore/jwt", "name": "adhocore/jwt",
@ -709,16 +709,16 @@
}, },
{ {
"name": "google/protobuf", "name": "google/protobuf",
"version": "v4.30.0", "version": "v4.30.1",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/protocolbuffers/protobuf-php.git", "url": "https://github.com/protocolbuffers/protobuf-php.git",
"reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24",
"reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -747,9 +747,9 @@
"proto" "proto"
], ],
"support": { "support": {
"source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1"
}, },
"time": "2025-03-04T22:54:49+00:00" "time": "2025-03-13T21:08:17+00:00"
}, },
{ {
"name": "jean85/pretty-package-versions", "name": "jean85/pretty-package-versions",
@ -3364,16 +3364,16 @@
}, },
{ {
"name": "utopia-php/abuse", "name": "utopia-php/abuse",
"version": "0.51.0", "version": "0.52.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/utopia-php/abuse.git", "url": "https://github.com/utopia-php/abuse.git",
"reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e" "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/utopia-php/abuse/zipball/661687b03277f1d202a0e8cf9da6e58c97da2b5e", "url": "https://api.github.com/repos/utopia-php/abuse/zipball/a0d6421e7e5baa3ac02755496dca9fdeaa814b93",
"reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e", "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -3381,7 +3381,7 @@
"ext-pdo": "*", "ext-pdo": "*",
"ext-redis": "*", "ext-redis": "*",
"php": ">=8.0", "php": ">=8.0",
"utopia-php/database": "0.60.*" "utopia-php/database": "0.*.*"
}, },
"require-dev": { "require-dev": {
"laravel/pint": "1.*", "laravel/pint": "1.*",
@ -3409,9 +3409,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/utopia-php/abuse/issues", "issues": "https://github.com/utopia-php/abuse/issues",
"source": "https://github.com/utopia-php/abuse/tree/0.51.0" "source": "https://github.com/utopia-php/abuse/tree/0.52.0"
}, },
"time": "2025-02-17T11:10:18+00:00" "time": "2025-03-06T03:48:29+00:00"
}, },
{ {
"name": "utopia-php/analytics", "name": "utopia-php/analytics",
@ -3461,21 +3461,21 @@
}, },
{ {
"name": "utopia-php/audit", "name": "utopia-php/audit",
"version": "0.54.0", "version": "0.55.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/utopia-php/audit.git", "url": "https://github.com/utopia-php/audit.git",
"reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975" "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/utopia-php/audit/zipball/1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", "url": "https://api.github.com/repos/utopia-php/audit/zipball/9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518",
"reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"php": ">=8.0", "php": ">=8.0",
"utopia-php/database": "0.60.*" "utopia-php/database": "0.*.*"
}, },
"require-dev": { "require-dev": {
"laravel/pint": "1.*", "laravel/pint": "1.*",
@ -3502,9 +3502,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/utopia-php/audit/issues", "issues": "https://github.com/utopia-php/audit/issues",
"source": "https://github.com/utopia-php/audit/tree/0.54.0" "source": "https://github.com/utopia-php/audit/tree/0.55.0"
}, },
"time": "2025-02-25T07:21:07+00:00" "time": "2025-03-06T03:47:47+00:00"
}, },
{ {
"name": "utopia-php/cache", "name": "utopia-php/cache",
@ -3705,16 +3705,16 @@
}, },
{ {
"name": "utopia-php/database", "name": "utopia-php/database",
"version": "0.60.6", "version": "0.61.1",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/utopia-php/database.git", "url": "https://github.com/utopia-php/database.git",
"reference": "f3c9aa964b39c6205069f038a26e709a15541406" "reference": "2e0165bd14a570ec151f400ed381108e81d15b94"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/f3c9aa964b39c6205069f038a26e709a15541406", "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94",
"reference": "f3c9aa964b39c6205069f038a26e709a15541406", "reference": "2e0165bd14a570ec151f400ed381108e81d15b94",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -3755,9 +3755,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/utopia-php/database/issues", "issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.60.6" "source": "https://github.com/utopia-php/database/tree/0.61.1"
}, },
"time": "2025-03-05T01:23:14+00:00" "time": "2025-03-14T01:19:38+00:00"
}, },
{ {
"name": "utopia-php/domains", "name": "utopia-php/domains",
@ -4159,16 +4159,16 @@
}, },
{ {
"name": "utopia-php/migration", "name": "utopia-php/migration",
"version": "0.6.20", "version": "0.6.22",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/utopia-php/migration.git", "url": "https://github.com/utopia-php/migration.git",
"reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba" "reference": "a0269746bd318ff0993f5aa008675b971689d5b5"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", "url": "https://api.github.com/repos/utopia-php/migration/zipball/a0269746bd318ff0993f5aa008675b971689d5b5",
"reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", "reference": "a0269746bd318ff0993f5aa008675b971689d5b5",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -4176,7 +4176,7 @@
"ext-curl": "*", "ext-curl": "*",
"ext-openssl": "*", "ext-openssl": "*",
"php": ">=8.1", "php": ">=8.1",
"utopia-php/database": "0.60.*", "utopia-php/database": "0.61.*",
"utopia-php/dsn": "0.2.*", "utopia-php/dsn": "0.2.*",
"utopia-php/framework": "0.33.*", "utopia-php/framework": "0.33.*",
"utopia-php/storage": "0.18.*" "utopia-php/storage": "0.18.*"
@ -4209,9 +4209,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/utopia-php/migration/issues", "issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/0.6.20" "source": "https://github.com/utopia-php/migration/tree/0.6.22"
}, },
"time": "2025-02-17T11:02:15+00:00" "time": "2025-03-13T07:35:55+00:00"
}, },
{ {
"name": "utopia-php/mongo", "name": "utopia-php/mongo",
@ -4810,16 +4810,16 @@
}, },
{ {
"name": "utopia-php/vcs", "name": "utopia-php/vcs",
"version": "0.9.3", "version": "0.9.4",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/utopia-php/vcs.git", "url": "https://github.com/utopia-php/vcs.git",
"reference": "865a00c67e81a20938b883f9aa802303790dd3b5" "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/865a00c67e81a20938b883f9aa802303790dd3b5", "url": "https://api.github.com/repos/utopia-php/vcs/zipball/1a8d280b176acc99ea8d9e7364b8767cbb206b4a",
"reference": "865a00c67e81a20938b883f9aa802303790dd3b5", "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -4854,9 +4854,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/utopia-php/vcs/issues", "issues": "https://github.com/utopia-php/vcs/issues",
"source": "https://github.com/utopia-php/vcs/tree/0.9.3" "source": "https://github.com/utopia-php/vcs/tree/0.9.4"
}, },
"time": "2025-02-26T16:33:35+00:00" "time": "2025-03-13T10:09:45+00:00"
}, },
{ {
"name": "utopia-php/websocket", "name": "utopia-php/websocket",
@ -8402,7 +8402,7 @@
], ],
"aliases": [], "aliases": [],
"minimum-stability": "stable", "minimum-stability": "stable",
"stability-flags": [], "stability-flags": {},
"prefer-stable": false, "prefer-stable": false,
"prefer-lowest": false, "prefer-lowest": false,
"platform": { "platform": {

13
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -387,6 +387,7 @@ class Deletes extends Action
$query = [ $query = [
Query::lessThan('accessedAt', $datetime), Query::lessThan('accessedAt', $datetime),
Query::orderDesc('accessedAt')
]; ];
$this->deleteByGroup( $this->deleteByGroup(
@ -420,6 +421,7 @@ class Deletes extends Action
// Delete Usage stats from projectDB // Delete Usage stats from projectDB
$this->deleteByGroup('stats', [ $this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::orderDesc('time'),
Query::equal('period', ['1h']), Query::equal('period', ['1h']),
], $dbForProject); ], $dbForProject);
@ -430,6 +432,7 @@ class Deletes extends Action
// Delete Usage stats from logsDB // Delete Usage stats from logsDB
$this->deleteByGroup('stats', [ $this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::orderDesc('time'),
Query::equal('period', ['1h']), Query::equal('period', ['1h']),
], $dbForLogs); ], $dbForLogs);
} }
@ -688,9 +691,11 @@ class Deletes extends Action
private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
{ {
$dbForProject = $getProjectDB($project); $dbForProject = $getProjectDB($project);
// Delete Executions // Delete Executions
$this->deleteByGroup('executions', [ $this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime) Query::lessThan('$createdAt', $datetime),
Query::orderDesc('$createdAt'),
], $dbForProject); ], $dbForProject);
} }
@ -708,7 +713,8 @@ class Deletes extends Action
// Delete Sessions // Delete Sessions
$this->deleteByGroup('sessions', [ $this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired) Query::lessThan('$createdAt', $expired),
Query::orderDesc('$createdAt'),
], $dbForProject); ], $dbForProject);
} }
@ -722,7 +728,8 @@ class Deletes extends Action
{ {
// Delete Dead Realtime Logs // Delete Dead Realtime Logs
$this->deleteByGroup('realtime', [ $this->deleteByGroup('realtime', [
Query::lessThan('timestamp', $datetime) Query::lessThan('timestamp', $datetime),
Query::orderDesc('timestamp'),
], $dbForPlatform); ], $dbForPlatform);
} }