From bbfd22ce2e68c86ee862ff450ee294d442554dec Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 17 Mar 2025 17:05:40 +0200 Subject: [PATCH 01/58] Index check for delete worker --- src/Appwrite/Platform/Workers/Deletes.php | 147 +++++++++++++++++----- 1 file changed, 113 insertions(+), 34 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..3c8bcdd635 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -248,7 +248,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('topicInternalId', [$topic->getInternalId()]) + Query::equal('topicInternalId', [$topic->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project) ); @@ -269,7 +270,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('targetInternalId', [$target->getInternalId()]) + Query::equal('targetInternalId', [$target->getInternalId()]), + Query::orderAsc(), ], $dbForProject, function (Document $subscriber) use ($dbForProject, $target) { @@ -303,10 +305,14 @@ class Deletes extends Action */ private function deleteExpiredTargets(Document $project, callable $getProjectDB): void { + /** + * todo: No index found for `expired` attribute + */ $this->deleteByGroup( 'targets', [ - Query::equal('expired', [true]) + Query::equal('expired', [true]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -317,10 +323,14 @@ class Deletes extends Action private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void { + /** + * todo: No index found for `sessionInternalId` attribute + */ $this->deleteByGroup( 'targets', [ - Query::equal('sessionInternalId', [$session->getInternalId()]) + Query::equal('sessionInternalId', [$session->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -348,10 +358,17 @@ class Deletes extends Action ); $query[] = Query::equal('resource', [$resource]); + if (!empty($resourceType)) { $query[] = Query::equal('resourceType', [$resourceType]); } + $query[] = Query::orderAsc(); + + /** + * todo: missing index on `resource`, `resourceType` + */ + $this->deleteByGroup( 'cache', $query, @@ -388,7 +405,7 @@ class Deletes extends Action $query = [ Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ]; $this->deleteByGroup( @@ -421,10 +438,10 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC ], $dbForProject); if ($project->getId() !== 'console') { @@ -433,10 +450,10 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC ], $dbForLogs); } } @@ -457,7 +474,8 @@ class Deletes extends Action $this->deleteByGroup( 'memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) + Query::equal('teamInternalId', [$teamInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $membership) use ($dbForProject) { @@ -547,7 +565,13 @@ class Deletes extends Action if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); } else { - $this->deleteByGroup($collection->getId(), [], database: $dbForProject); + $this->deleteByGroup( + $collection->getId(), + [ + Query::orderAsc() + ], + database: $dbForProject + ); } } catch (Throwable $e) { Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); @@ -567,58 +591,82 @@ class Deletes extends Action // Delete Platforms $this->deleteByGroup('platforms', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete project and function rules $this->deleteByGroup('rules', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); // Delete Keys $this->deleteByGroup('keys', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete Webhooks $this->deleteByGroup('webhooks', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Installations $this->deleteByGroup('installations', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Repositories $this->deleteByGroup('repositories', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS comments $this->deleteByGroup('vcsComments', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); - // Delete Schedules (No projectInternalId in this collection) + /** + * No projectInternalId in this collection + * todo: No index for `projectId` attribute + */ + // Delete Schedules () $this->deleteByGroup('schedules', [ Query::equal('projectId', [$projectId]), + Query::orderAsc() ], $dbForPlatform); // Delete metadata table if ($projectTables) { $dbForProject->deleteCollection(Database::METADATA); } elseif ($sharedTablesV1) { - $this->deleteByGroup(Database::METADATA, [], $dbForProject); + $this->deleteByGroup( + Database::METADATA, + [ + Query::orderAsc() + ], + $dbForProject + ); } elseif ($sharedTablesV2) { $queries = \array_map( fn ($id) => Query::notEqual('$id', $id), $projectCollectionIds ); - $this->deleteByGroup(Database::METADATA, $queries, $dbForProject); + $queries[] = Query::orderAsc(); + + $this->deleteByGroup( + Database::METADATA, + $queries, + $dbForProject + ); } // Delete all storage directories @@ -643,14 +691,16 @@ class Deletes extends Action // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); $dbForProject->purgeCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($dbForProject) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); @@ -663,19 +713,25 @@ class Deletes extends Action // Delete tokens $this->deleteByGroup('tokens', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete identities + /** + * todo: Remove Duplication index `_key_userInternalId` + */ $this->deleteByGroup('identities', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete targets $this->deleteByGroup( 'targets', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $target) use ($getProjectDB, $project) { @@ -699,7 +755,7 @@ class Deletes extends Action $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -719,7 +775,7 @@ class Deletes extends Action $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -735,7 +791,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderDesc('$internalId'), + Query::orderAsc(), // KEY "_key_timestamp" ("timestamp" DESC), ], $dbForPlatform); } @@ -755,7 +811,7 @@ class Deletes extends Action $this->deleteByGroup(Audit::COLLECTION, [ Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderDesc('$internalId'), + Query::orderAsc(), // KEY "index-time" ("time" DESC) ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); @@ -780,33 +836,39 @@ class Deletes extends Action /** * Delete rules + * todo: No index for this query */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), - Query::equal('projectInternalId', [$project->getInternalId()]) + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); /** * Delete Variables + * todo: No index for this query */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ Query::equal('resourceType', ['function']), - Query::equal('resourceInternalId', [$functionInternalId]) + Query::equal('resourceInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject); /** * Delete Deployments + * todo: No index for `resourceInternalId`, perhaps use resourceId until fixed */ Console::info("Deleting deployments for function " . $functionId); $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ - Query::equal('resourceInternalId', [$functionInternalId]) + Query::equal('resourceInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $this->deleteDeploymentFiles($deviceForFunctions, $document); @@ -814,12 +876,14 @@ class Deletes extends Action /** * Delete builds + * todo: No index for `deploymentInternalId`, perhaps use deploymentId until fixed */ Console::info("Deleting builds for function " . $functionId); foreach ($deploymentInternalIds as $deploymentInternalId) { $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -827,26 +891,35 @@ class Deletes extends Action /** * Delete Executions + * todo: No index for `functionInternalId` , perhaps use functionId until fixed */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ - Query::equal('functionInternalId', [$functionInternalId]) + Query::equal('functionInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject); /** * Delete VCS Repositories and VCS Comments + * todo: no index for this query */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform) { $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $projectInternalId = $document->getAttribute('projectInternalId', ''); + + /** + * todo: add index to this query + */ $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); }); @@ -942,11 +1015,13 @@ class Deletes extends Action /** * Delete builds + * todo: no index for `deploymentInternalId` change to temporary to deploymentId? */ Console::info("Deleting builds for deployment " . $deploymentId); $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -974,6 +1049,10 @@ class Deletes extends Action ): void { $start = \microtime(true); + /** + * deleteDocuments uses a cursor, we need to add a unique order by field or use default + */ + try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From 5f7ceecb30e64f11fa6bb0a5cb746007cc983b8d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Mar 2025 03:46:14 +0000 Subject: [PATCH 02/58] chore: stop tests on failure --- phpunit.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpunit.xml b/phpunit.xml index 4c4e55ea4e..598b730908 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > From 795928a037644ca8aca6a3ad6fd07b2f6ffffef5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 18 Mar 2025 16:59:05 +0200 Subject: [PATCH 03/58] Fix $id --- src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 3c8bcdd635..a14e54caab 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1045,7 +1045,8 @@ class Deletes extends Action string $collection, array $queries, Database $database, - ?callable $callback = null + ?callable $callback = null, + bool $shortSelect = false ): void { $start = \microtime(true); @@ -1053,6 +1054,12 @@ class Deletes extends Action * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ + if (!\is_callable($callback) && $shortSelect) { + $queries = array_merge($queries, [ + Query::select(['$internalId', '$id', '$permissions', '$updatedAt']) + ]); + } + try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From cc0a97b10e30290fce85fc1a84cf5f9948c8f391 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 19 Mar 2025 16:37:15 +0200 Subject: [PATCH 04/58] Fixes --- app/config/collections/common.php | 6 +++--- src/Appwrite/Platform/Workers/Deletes.php | 24 ++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/app/config/collections/common.php b/app/config/collections/common.php index f68400e226..3f6245dfdd 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1038,7 +1038,7 @@ return [ '$id' => ID::custom('providerUid'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 2048, // Decrease to 128 as in index length? 'signed' => true, 'required' => false, 'default' => null, @@ -1107,14 +1107,14 @@ return [ '$id' => ID::custom('_key_userInternalId_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['userInternalId', 'provider', 'providerUid'], - 'lengths' => [11, 128, 128], + 'lengths' => [11, 128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [128, 128], + 'lengths' => [128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index a14e54caab..27ff27fe31 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -357,21 +357,23 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query[] = Query::equal('resource', [$resource]); + $queries = [ + Query::equal('resource', [$resource]) + ]; if (!empty($resourceType)) { - $query[] = Query::equal('resourceType', [$resourceType]); + $queries[] = Query::equal('resourceType', [$resourceType]); } - $query[] = Query::orderAsc(); + $queries[] = Query::orderAsc(); /** - * todo: missing index on `resource`, `resourceType` + * todo: No index on `resource`, `resourceType`, But it is fine since resource index is a strong index! */ $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -719,7 +721,7 @@ class Deletes extends Action // Delete identities /** - * todo: Remove Duplication index `_key_userInternalId` + * todo: Remove Duplication index `_key_userInternalId` , But lets leave because of index length issue in unique index */ $this->deleteByGroup('identities', [ Query::equal('userInternalId', [$userInternalId]), @@ -836,13 +838,13 @@ class Deletes extends Action /** * Delete rules - * todo: No index for this query + * todo: No index for this query, drop _key_projectInternalId and create _key_projectInternalId, resourceInternalId, resourceType */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ - Query::equal('resourceType', ['function']), - Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]), + Query::equal('resourceInternalId', [$functionInternalId]), + Query::equal('resourceType', ['function']), Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); @@ -850,12 +852,12 @@ class Deletes extends Action /** * Delete Variables - * todo: No index for this query + * todo: No index for this query , drop _key_resourceInternalId and create new one with {resourceInternalId, resourceType} */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ - Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), + Query::equal('resourceType', ['function']), Query::orderAsc() ], $dbForProject); From 9c54cce5bd8c61d26a549273dfe30b6bcfd94751 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 20 Mar 2025 09:07:32 +0200 Subject: [PATCH 05/58] Queries fixes --- src/Appwrite/Platform/Workers/Deletes.php | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 27ff27fe31..b7fce0405d 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -365,6 +365,7 @@ class Deletes extends Action $queries[] = Query::equal('resourceType', [$resourceType]); } + $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); $queries[] = Query::orderAsc(); /** @@ -404,7 +405,8 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query = [ + $queries = [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -412,7 +414,7 @@ class Deletes extends Action $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -440,6 +442,7 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -452,6 +455,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -755,6 +759,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -775,6 +780,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -811,6 +817,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), // KEY "index-time" ("time" DESC) @@ -897,6 +904,7 @@ class Deletes extends Action */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('functionInternalId', [$functionInternalId]), Query::orderAsc() ], $dbForProject); @@ -1017,7 +1025,7 @@ class Deletes extends Action /** * Delete builds - * todo: no index for `deploymentInternalId` change to temporary to deploymentId? + * todo: no index for `deploymentInternalId` Same as above index , no need to handle again... */ Console::info("Deleting builds for deployment " . $deploymentId); @@ -1047,8 +1055,7 @@ class Deletes extends Action string $collection, array $queries, Database $database, - ?callable $callback = null, - bool $shortSelect = false + ?callable $callback = null ): void { $start = \microtime(true); @@ -1056,12 +1063,6 @@ class Deletes extends Action * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ - if (!\is_callable($callback) && $shortSelect) { - $queries = array_merge($queries, [ - Query::select(['$internalId', '$id', '$permissions', '$updatedAt']) - ]); - } - try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From 5eb7c36f50e2713aee76beaff1bb207eca130116 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 10:32:36 +0000 Subject: [PATCH 06/58] chore: add console audit retention --- .env | 2 +- app/worker.php | 2 +- docker-compose.yml | 1 + src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 1893e023ba..c68f0a0d52 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/worker.php b/app/worker.php index eeefe80000..5d7bd4de41 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,7 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); + return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index b6dc80df6a..af6a323ec8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,6 +389,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES + - _APP_MAINTENANCE_RETENTION_AUDIT appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..75380ed3fe 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -751,9 +751,16 @@ class Deletes extends Action $projectId = $project->getId(); $dbForProject = $getProjectDB($project); + [$projectAuditRetention, $consoleAuditRetention] = array_map( + function ($part) { + return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); + }, + explode(',', $auditRetention) + ); + try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', $auditRetention), + Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 9d8289221925ddf7eeebbd42f56ba9be311dd2a0 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 11:32:06 +0000 Subject: [PATCH 07/58] chore: update logic --- app/worker.php | 10 +++++++++- src/Appwrite/Platform/Workers/Deletes.php | 15 ++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/app/worker.php b/app/worker.php index 5d7bd4de41..18b51eda25 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,15 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months + return array_map( + function ($part) { + [$key, $value] = explode('=', $part); + return [ + $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) + ]; + }, + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + ); }); Server::setResource('executionRetention', function () { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 75380ed3fe..93f8748e49 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log) => $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -746,21 +746,14 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void + private function deleteAuditLogs(Document $project, callable $getProjectDB, array $auditRetention): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - [$projectAuditRetention, $consoleAuditRetention] = array_map( - function ($part) { - return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); - }, - explode(',', $auditRetention) - ); - try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), + Query::lessThan('time', ($projectId === 'console' ? $auditRetention['console'] : $auditRetention['project'])), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 7c3f8504bb65e598f50d07912b5d753c86f5e1f6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:15:14 +0000 Subject: [PATCH 08/58] chore: fix initialization --- app/worker.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/worker.php b/app/worker.php index 18b51eda25..e305ec42d6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,14 +219,14 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_map( - function ($part) { + return array_reduce( + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + function ($carry, $part) { [$key, $value] = explode('=', $part); - return [ - $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) - ]; + $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); + return $carry; }, - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + [] ); }); From b0b57935da0e54074766a3f9a144f41fe2081a55 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:42:19 +0000 Subject: [PATCH 09/58] chore: add comments --- app/worker.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/worker.php b/app/worker.php index e305ec42d6..d0fa5cf4f8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -215,12 +215,12 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { }, ['pools', 'cache']); Server::setResource('abuseRetention', function () { - return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); + return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day }); Server::setResource('auditRetention', function () { return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months function ($carry, $part) { [$key, $value] = explode('=', $part); $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); @@ -231,7 +231,7 @@ Server::setResource('auditRetention', function () { }); Server::setResource('executionRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); // 14 days }); Server::setResource('cache', function (Registry $register) { From 13750f7dbd5b5850075eefab4bacf3e782752004 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:42:10 +0000 Subject: [PATCH 10/58] feat: introduce new env --- .env | 3 ++- app/config/variables.php | 11 ++++++++++- app/views/install/compose.phtml | 3 +++ app/worker.php | 13 ++++--------- docker-compose.yml | 3 +++ 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.env b/.env index c68f0a0d52..c10c12613b 100644 --- a/.env +++ b/.env @@ -85,7 +85,8 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 +_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/config/variables.php b/app/config/variables.php index 98dd9ffec1..27463d2fee 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1048,13 +1048,22 @@ return [ ], [ 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT', - 'description' => 'IThe maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', + 'description' => 'The maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', 'introduction' => '0.7.0', 'default' => '1209600', 'required' => false, 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', + 'description' => 'The maximum duration (in seconds) upto which to retain console audit logs. The default value is 15778800 seconds (6 months).', + 'introduction' => '1.6.2', + 'default' => '15778800', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_MAINTENANCE_RETENTION_ABUSE', 'description' => 'The maximum duration (in seconds) upto which to retain abuse logs. The default value is 86400 seconds (1 day).', diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index fd05d2a0b6..7dfe14fcef 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -148,6 +148,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -340,6 +341,7 @@ $image = $this->getParam('image', ''); - _APP_EXECUTOR_HOST - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES @@ -651,6 +653,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES diff --git a/app/worker.php b/app/worker.php index d0fa5cf4f8..3f99089eef 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,15 +219,10 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months - function ($carry, $part) { - [$key, $value] = explode('=', $part); - $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); - return $carry; - }, - [] - ); + return [ + 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + ]; }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index af6a323ec8..05ddba967a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,6 +171,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -390,6 +391,7 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE appwrite-worker-databases: entrypoint: worker-databases @@ -722,6 +724,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY From 7c795da72fd7aa05b280f27012f87357a08d2814 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:48:50 +0000 Subject: [PATCH 11/58] chore: fix type --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 3f99089eef..744c758c69 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) ]; }); From c0c0c921dc249f2f235a9bd7e8efdce66ae3c137 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 08:08:30 +0000 Subject: [PATCH 12/58] chore: fix naming --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 744c758c69..491f266a18 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), // 14 days + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) // 6 months ]; }); From d661195a5f4bf375f7b72e5629e1a1dfd3fe8d3d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 15:59:30 +0000 Subject: [PATCH 13/58] chore: only update total count for prviledged users --- app/controllers/api/teams.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index b45c9fd3b9..dfdccf5e99 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -615,7 +615,10 @@ App::post('/v1/teams/:teamId/memberships') $membership = ($isPrivilegedUser || $isAppUser) ? Authorization::skip(fn () => $dbForProject->createDocument('memberships', $membership)) : $dbForProject->createDocument('memberships', $membership); - Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); + + if ($isPrivilegedUser || $isAppUser) { + Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); + } } elseif ($membership->getAttribute('confirm') === false) { $membership->setAttribute('secret', Auth::hash($secret)); From 7c2ab4fcf652b2823a42c04daf8ce4c0233eb609 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 22 Mar 2025 13:47:09 +0530 Subject: [PATCH 14/58] update: change sample rate for 4xx via env. --- app/controllers/general.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index a374aa3c39..1423763cec 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -871,17 +871,18 @@ App::error() if (!empty($providerConfig) && $error->getCode() >= 400 && $error->getCode() < 500) { // Register error logger try { - $loggingProvider = new DSN($providerConfig ?? ''); + $loggingProvider = new DSN($providerConfig); $providerName = $loggingProvider->getScheme(); if (!empty($providerName) && $providerName === 'sentry') { $key = $loggingProvider->getPassword(); $projectId = $loggingProvider->getUser() ?? ''; $host = 'https://' . $loggingProvider->getHost(); + $sampleRate = $loggingProvider->getParam('sample', 0.01); $adapter = new Sentry($projectId, $key, $host); $logger = new Logger($adapter); - $logger->setSample(0.01); + $logger->setSample($sampleRate); $publish = true; } else { throw new \Exception('Invalid experimental logging provider'); From 86856bf87d1afa58543411eda105fb975009e5db Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 22 Mar 2025 13:55:23 +0000 Subject: [PATCH 15/58] chore: update gitpod.yml config --- .gitpod.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 478b62fc8d..e9c0928229 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,6 +1,7 @@ tasks: - name: Run Appwrite Docker Stack init: | + git config --global --add safe.directory /app git submodule update --init docker compose build docker compose pull @@ -23,11 +24,3 @@ vscode: extensions: - ms-azuretools.vscode-docker - zobo.php-intellisense - -github: - # https://www.gitpod.io/docs/prebuilds#github-specific-configuration - prebuilds: - # enable for pull requests coming from forks (defaults to false) - pullRequestsFromForks: true - # add a check to pull requests (defaults to true) - addCheck: false From 37dfe0d719b49f32b2540b3d6b3310fcd110e976 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 22 Mar 2025 14:05:39 +0000 Subject: [PATCH 16/58] refactor: initialization of audit retention --- app/worker.php | 14 ++++++-------- src/Appwrite/Platform/Workers/Deletes.php | 10 +++++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/app/worker.php b/app/worker.php index 491f266a18..90496c0430 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,8 +16,6 @@ use Appwrite\Event\Migration; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; -/** remove */ -/** /remove */ use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Swoole\Runtime; @@ -218,12 +216,12 @@ Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day }); -Server::setResource('auditRetention', function () { - return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), // 14 days - 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) // 6 months - ]; -}); +Server::setResource('auditRetention', function (Document $project) { + if ($project->getId() === 'console') { + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)); // 6 months + } + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); // 14 days +}, ['project']); Server::setResource('executionRetention', function () { return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); // 14 days diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 93f8748e49..bb5cc81fcc 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log) => + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -742,18 +742,18 @@ class Deletes extends Action /** * @param Database $dbForPlatform * @param callable $getProjectDB - * @param string $datetime + * @param string $auditRetention * @return void * @throws Exception */ - private function deleteAuditLogs(Document $project, callable $getProjectDB, array $auditRetention): void + private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', ($projectId === 'console' ? $auditRetention['console'] : $auditRetention['project'])), + Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 26821c83726db7506fbf2d0828f34a908cb5d1d3 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 11:08:39 +0200 Subject: [PATCH 17/58] Bump database 0.62 --- composer.json | 2 +- composer.lock | 311 ++++---------------------------------------------- 2 files changed, 22 insertions(+), 291 deletions(-) diff --git a/composer.json b/composer.json index d7b8505b5c..b1b6aed539 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.61.*", + "utopia-php/database": "0.62.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index cdba756c72..1ea887fdbc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e0d7f21b681e4591144fec16c4f0d6aa", + "content-hash": "d2bec8137dcd84994121f89a29932d31", "packages": [ { "name": "adhocore/jwt", @@ -751,65 +751,6 @@ }, "time": "2025-03-13T21:08:17+00:00" }, - { - "name": "jean85/pretty-package-versions", - "version": "2.1.0", - "source": { - "type": "git", - "url": "https://github.com/Jean85/pretty-package-versions.git", - "reference": "3c4e5f62ba8d7de1734312e4fff32f67a8daaf10" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/3c4e5f62ba8d7de1734312e4fff32f67a8daaf10", - "reference": "3c4e5f62ba8d7de1734312e4fff32f67a8daaf10", - "shasum": "" - }, - "require": { - "composer-runtime-api": "^2.1.0", - "php": "^7.4|^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.2", - "jean85/composer-provided-replaced-stub-package": "^1.0", - "phpstan/phpstan": "^1.4", - "phpunit/phpunit": "^7.5|^8.5|^9.6", - "vimeo/psalm": "^4.3 || ^5.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.x-dev" - } - }, - "autoload": { - "psr-4": { - "Jean85\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Alessandro Lai", - "email": "alessandro.lai85@gmail.com" - } - ], - "description": "A library to get pretty versions strings of installed dependencies", - "keywords": [ - "composer", - "package", - "release", - "versions" - ], - "support": { - "issues": "https://github.com/Jean85/pretty-package-versions/issues", - "source": "https://github.com/Jean85/pretty-package-versions/tree/2.1.0" - }, - "time": "2024-11-18T16:19:46+00:00" - }, { "name": "league/csv", "version": "9.14.0", @@ -968,75 +909,6 @@ }, "time": "2023-10-02T10:01:54+00:00" }, - { - "name": "mongodb/mongodb", - "version": "1.10.0", - "source": { - "type": "git", - "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "b0bbd657f84219212487d01a8ffe93a789e1e488" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/b0bbd657f84219212487d01a8ffe93a789e1e488", - "reference": "b0bbd657f84219212487d01a8ffe93a789e1e488", - "shasum": "" - }, - "require": { - "ext-hash": "*", - "ext-json": "*", - "ext-mongodb": "^1.11.0", - "jean85/pretty-package-versions": "^1.2 || ^2.0.1", - "php": "^7.1 || ^8.0", - "symfony/polyfill-php80": "^1.19" - }, - "require-dev": { - "doctrine/coding-standard": "^9.0", - "squizlabs/php_codesniffer": "^3.6", - "symfony/phpunit-bridge": "^5.2" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.10.x-dev" - } - }, - "autoload": { - "files": [ - "src/functions.php" - ], - "psr-4": { - "MongoDB\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Andreas Braun", - "email": "andreas.braun@mongodb.com" - }, - { - "name": "Jeremy Mikola", - "email": "jmikola@gmail.com" - } - ], - "description": "MongoDB driver library", - "homepage": "https://jira.mongodb.org/browse/PHPLIB", - "keywords": [ - "database", - "driver", - "mongodb", - "persistence" - ], - "support": { - "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.10.0" - }, - "time": "2021-10-20T22:22:37+00:00" - }, { "name": "mustangostang/spyc", "version": "0.6.3", @@ -2371,16 +2243,16 @@ }, { "name": "ramsey/collection", - "version": "2.1.0", + "version": "2.1.1", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "url": "https://api.github.com/repos/ramsey/collection/zipball/344572933ad0181accbf4ba763e85a0306a8c5e2", + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2", "shasum": "" }, "require": { @@ -2441,9 +2313,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.1.0" + "source": "https://github.com/ramsey/collection/tree/2.1.1" }, - "time": "2025-03-02T04:48:29+00:00" + "time": "2025-03-22T05:38:12+00:00" }, { "name": "ramsey/uuid", @@ -2932,86 +2804,6 @@ ], "time": "2024-09-09T11:45:10+00:00" }, - { - "name": "symfony/polyfill-php80", - "version": "v1.31.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/60328e362d4c2c802a54fcbf04f9d3fb892b4cf8", - "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8", - "shasum": "" - }, - "require": { - "php": ">=7.2" - }, - "type": "library", - "extra": { - "thanks": { - "url": "https://github.com/symfony/polyfill", - "name": "symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Php80\\": "" - }, - "classmap": [ - "Resources/stubs" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Ion Bazan", - "email": "ion.bazan@gmail.com" - }, - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.31.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2024-09-09T11:45:10+00:00" - }, { "name": "symfony/polyfill-php82", "version": "v1.31.0", @@ -3705,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.2", + "version": "0.62.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" + "reference": "65dc51466c12552add10395900cdbb4728da4068" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", + "url": "https://api.github.com/repos/utopia-php/database/zipball/65dc51466c12552add10395900cdbb4728da4068", + "reference": "65dc51466c12552add10395900cdbb4728da4068", "shasum": "" }, "require": { @@ -3722,8 +3514,7 @@ "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.12.*", - "utopia-php/framework": "0.33.*", - "utopia-php/mongo": "0.3.*" + "utopia-php/framework": "0.33.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -3755,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.2" + "source": "https://github.com/utopia-php/database/tree/0.62.1" }, - "time": "2025-03-15T11:47:42+00:00" + "time": "2025-03-24T08:27:18+00:00" }, { "name": "utopia-php/domains", @@ -4159,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.1", + "version": "0.8.2", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab" + "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", - "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", + "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", "shasum": "" }, "require": { @@ -4176,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.61.*", + "utopia-php/database": "0.62.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4209,69 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.1" + "source": "https://github.com/utopia-php/migration/tree/0.8.2" }, - "time": "2025-03-18T07:48:08+00:00" - }, - { - "name": "utopia-php/mongo", - "version": "0.3.1", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/mongo.git", - "reference": "52326a9a43e2d27ff0c15c48ba746dacbe9a7aee" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/52326a9a43e2d27ff0c15c48ba746dacbe9a7aee", - "reference": "52326a9a43e2d27ff0c15c48ba746dacbe9a7aee", - "shasum": "" - }, - "require": { - "ext-mongodb": "*", - "mongodb/mongodb": "1.10.0", - "php": ">=8.0" - }, - "require-dev": { - "fakerphp/faker": "^1.14", - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.4", - "swoole/ide-helper": "4.8.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\Mongo\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Wess", - "email": "wess@appwrite.io" - } - ], - "description": "A simple library to manage Mongo database", - "keywords": [ - "database", - "mongo", - "php", - "upf", - "utopia" - ], - "support": { - "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/0.3.1" - }, - "time": "2023-09-01T17:25:28+00:00" + "time": "2025-03-24T09:05:31+00:00" }, { "name": "utopia-php/orchestration", From 5479e6f8f1f51691e168564b2592d15df917a421 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 18 Mar 2025 12:29:32 +0100 Subject: [PATCH 18/58] chore(audits): return queue pre-fetch results --- src/Appwrite/Platform/Workers/Audits.php | 49 +++++++++++++----------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index ed5ff8010a..962bc622d1 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,12 +12,12 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Queue\Result\Commit; +use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,9 +27,7 @@ class Audits extends Action protected function getBatchSize(): int { - return System::getEnv('_APP_ENV', 'development') === 'development' - ? self::BATCH_SIZE_DEVELOPMENT - : self::BATCH_SIZE_PRODUCTION; + return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); } public static function getName(): string @@ -57,13 +55,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return void + * @return Commit|NoCommit * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): void + public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit { $payload = $message->getPayload() ?? []; @@ -119,29 +117,34 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $shouldProcessBatch = \count($this->logs) >= $batchSize; - if (!$shouldProcessBatch && \count($this->logs) > 0) { + $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs), 0); + $shouldProcessBatch = $logCount >= $batchSize; + if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if ($shouldProcessBatch) { - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + if (!$shouldProcessBatch) { + return new NoCommit(); + } - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); - unset($this->logs[$internalId]); - } - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - } finally { - $this->lastTriggeredTime = time(); + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); + + unset($this->logs[$internalId]); } + return new Commit(); + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + return new NoCommit(); + } finally { + $this->lastTriggeredTime = time(); } } } From 25b6d59e3071b6d223cb53c8e9453631d5f3ec43 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 12:54:19 +0200 Subject: [PATCH 19/58] Add internalId --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index b8cff64214..d1b143ec9b 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -60,11 +60,19 @@ class Base extends Queries 'type' => Database::VAR_STRING, 'array' => false, ]); + + $attributes[] = new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); + $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, 'array' => false, ]); + $attributes[] = new Document([ 'key' => '$updatedAt', 'type' => Database::VAR_DATETIME, From 42f72b4477ef951aa131429f4f513579b2a4cf46 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 13:33:36 +0200 Subject: [PATCH 20/58] Add internalId to orders --- .../Utopia/Database/Validator/Queries/Base.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index d1b143ec9b..85fbf43e66 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -61,12 +61,6 @@ class Base extends Queries 'array' => false, ]); - $attributes[] = new Document([ - 'key' => '$internalId', - 'type' => Database::VAR_STRING, - 'array' => false, - ]); - $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, @@ -84,8 +78,15 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order($attributes), - ]; + new Order( + array_merge($attributes, [ + new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]) + ]) + )]; parent::__construct($validators); } From 0648de2046c3b7561d37877b3b06c8c811ddc06d Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 14:32:57 +0200 Subject: [PATCH 21/58] Revert Base --- .../Utopia/Database/Validator/Queries/Base.php | 13 ++----------- .../Databases/DatabasesCustomServerTest.php | 4 ++-- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 85fbf43e66..b8cff64214 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -60,13 +60,11 @@ class Base extends Queries 'type' => Database::VAR_STRING, 'array' => false, ]); - $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, 'array' => false, ]); - $attributes[] = new Document([ 'key' => '$updatedAt', 'type' => Database::VAR_DATETIME, @@ -78,15 +76,8 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order( - array_merge($attributes, [ - new Document([ - 'key' => '$internalId', - 'type' => Database::VAR_STRING, - 'array' => false, - ]) - ]) - )]; + new Order($attributes), + ]; parent::__construct($validators); } diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 57e0b93634..70f8bea4f2 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -120,7 +120,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->toString(), + Query::orderDesc()->setAttribute('')->toString(), ], ]); @@ -453,7 +453,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->toString(), + Query::orderDesc()->setAttribute('')->toString(), ], ]); From e11e7e6780cbf8a3dc2ac10c4302360b7d4963c5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:33:02 +0200 Subject: [PATCH 22/58] Add internalId --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 8 +++++++- .../e2e/Services/Databases/DatabasesCustomServerTest.php | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index b8cff64214..a13314ab2f 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -71,12 +71,18 @@ class Base extends Queries 'array' => false, ]); + $internalId = new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); + $validators = [ new Limit(), new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order($attributes), + new Order(array_merge($attributes, [$internalId])), ]; parent::__construct($validators); diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 70f8bea4f2..55cb17b9f8 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -123,7 +123,7 @@ class DatabasesCustomServerTest extends Scope Query::orderDesc()->setAttribute('')->toString(), ], ]); - + var_dump($databases); $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); From 5799ca1041b8f945c45285f557a35ef6b8633381 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:37:34 +0200 Subject: [PATCH 23/58] remove var_dump --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 55cb17b9f8..206567e789 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -120,10 +120,9 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->setAttribute('')->toString(), + Query::orderDesc()->toString(), ], ]); - var_dump($databases); $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); @@ -453,7 +452,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->setAttribute('')->toString(), + Query::orderDesc()->toString(), ], ]); From 2795505e8288074f8981edb90e5f3279bfaf9938 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:38:01 +0200 Subject: [PATCH 24/58] line --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 206567e789..57e0b93634 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -123,6 +123,7 @@ class DatabasesCustomServerTest extends Scope Query::orderDesc()->toString(), ], ]); + $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); From c0cb4b26a3ebf73822f0abea8c6aeae9dd4d8875 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:51:47 +1300 Subject: [PATCH 25/58] Apply suggestions from code review --- src/Appwrite/Platform/Workers/Deletes.php | 35 ++++------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8deb227c78..0de79cd67e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -305,9 +305,6 @@ class Deletes extends Action */ private function deleteExpiredTargets(Document $project, callable $getProjectDB): void { - /** - * todo: No index found for `expired` attribute - */ $this->deleteByGroup( 'targets', [ @@ -323,9 +320,6 @@ class Deletes extends Action private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void { - /** - * todo: No index found for `sessionInternalId` attribute - */ $this->deleteByGroup( 'targets', [ @@ -368,10 +362,6 @@ class Deletes extends Action $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); $queries[] = Query::orderAsc(); - /** - * todo: No index on `resource`, `resourceType`, But it is fine since resource index is a strong index! - */ - $this->deleteByGroup( 'cache', $queries, @@ -446,7 +436,7 @@ class Deletes extends Action Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC + Query::orderDesc(), ], $dbForProject); if ($project->getId() !== 'console') { @@ -459,7 +449,7 @@ class Deletes extends Action Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC + Query::orderDesc(), ], $dbForLogs); } } @@ -639,11 +629,7 @@ class Deletes extends Action Query::orderAsc() ], $dbForPlatform); - /** - * No projectInternalId in this collection - * todo: No index for `projectId` attribute - */ - // Delete Schedules () + // Delete Schedules $this->deleteByGroup('schedules', [ Query::equal('projectId', [$projectId]), Query::orderAsc() @@ -724,9 +710,6 @@ class Deletes extends Action ], $dbForProject); // Delete identities - /** - * todo: Remove Duplication index `_key_userInternalId` , But lets leave because of index length issue in unique index - */ $this->deleteByGroup('identities', [ Query::equal('userInternalId', [$userInternalId]), Query::orderAsc() @@ -799,7 +782,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderAsc(), // KEY "_key_timestamp" ("timestamp" DESC), + Query::orderAsc(), ], $dbForPlatform); } @@ -820,7 +803,7 @@ class Deletes extends Action Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderAsc(), // KEY "index-time" ("time" DESC) + Query::orderAsc(), ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); @@ -845,7 +828,6 @@ class Deletes extends Action /** * Delete rules - * todo: No index for this query, drop _key_projectInternalId and create _key_projectInternalId, resourceInternalId, resourceType */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ @@ -859,7 +841,6 @@ class Deletes extends Action /** * Delete Variables - * todo: No index for this query , drop _key_resourceInternalId and create new one with {resourceInternalId, resourceType} */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ @@ -885,7 +866,6 @@ class Deletes extends Action /** * Delete builds - * todo: No index for `deploymentInternalId`, perhaps use deploymentId until fixed */ Console::info("Deleting builds for function " . $functionId); @@ -900,7 +880,6 @@ class Deletes extends Action /** * Delete Executions - * todo: No index for `functionInternalId` , perhaps use functionId until fixed */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ @@ -923,9 +902,6 @@ class Deletes extends Action $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $projectInternalId = $document->getAttribute('projectInternalId', ''); - /** - * todo: add index to this query - */ $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('projectInternalId', [$projectInternalId]), @@ -1025,7 +1001,6 @@ class Deletes extends Action /** * Delete builds - * todo: no index for `deploymentInternalId` Same as above index , no need to handle again... */ Console::info("Deleting builds for deployment " . $deploymentId); From a20140eb36dfc7238324cf40989bab9842e89abf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:52:54 +1300 Subject: [PATCH 26/58] Update src/Appwrite/Platform/Workers/Deletes.php --- src/Appwrite/Platform/Workers/Deletes.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 0de79cd67e..93dbaf501c 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -851,7 +851,6 @@ class Deletes extends Action /** * Delete Deployments - * todo: No index for `resourceInternalId`, perhaps use resourceId until fixed */ Console::info("Deleting deployments for function " . $functionId); From c9c9db564053523be084bb5a97832221eb6ee28c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:52:59 +1300 Subject: [PATCH 27/58] Update src/Appwrite/Platform/Workers/Deletes.php --- src/Appwrite/Platform/Workers/Deletes.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 93dbaf501c..6dff127784 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -889,7 +889,6 @@ class Deletes extends Action /** * Delete VCS Repositories and VCS Comments - * todo: no index for this query */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ From 6bcdc8c769185b77dc9689ed691e53a4071b1453 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 16:01:27 +1300 Subject: [PATCH 28/58] Update src/Appwrite/Utopia/Database/Validator/Queries/Base.php --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index a13314ab2f..664476af36 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -82,7 +82,7 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order(array_merge($attributes, [$internalId])), + new Order([...$attributes, $internalId])), ]; parent::__construct($validators); From 74d9a3c181ebf09b9a7b6da710b6431dcaac4097 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 16:19:18 +1300 Subject: [PATCH 29/58] Fix syntax --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 664476af36..e8eafba5a0 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -82,7 +82,7 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order([...$attributes, $internalId])), + new Order([...$attributes, $internalId]), ]; parent::__construct($validators); From 6195674611b22d8d65c83a038f2d28cdc662ba36 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 21:20:26 +1300 Subject: [PATCH 30/58] Update database --- composer.json | 2 +- composer.lock | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/composer.json b/composer.json index b1b6aed539..5b0f26b883 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.62.*", + "utopia-php/database": "dev-feat-multi-tenant-insert as 0.62.1", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 1ea887fdbc..09b8250af6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d2bec8137dcd84994121f89a29932d31", + "content-hash": "640d2a8be42ae521ea12d8a908873f2b", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.62.1", + "version": "dev-feat-multi-tenant-insert", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "65dc51466c12552add10395900cdbb4728da4068" + "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/65dc51466c12552add10395900cdbb4728da4068", - "reference": "65dc51466c12552add10395900cdbb4728da4068", + "url": "https://api.github.com/repos/utopia-php/database/zipball/2e01c999153a202a8bae9c21e4302e7a674442d0", + "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.62.1" + "source": "https://github.com/utopia-php/database/tree/feat-multi-tenant-insert" }, - "time": "2025-03-24T08:27:18+00:00" + "time": "2025-03-25T04:45:25+00:00" }, { "name": "utopia-php/domains", @@ -8132,9 +8132,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-multi-tenant-insert", + "alias": "0.62.1", + "alias_normalized": "0.62.1.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From d89cdbcf5a52717b7dc7e48b88020c2604502f0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 25 Mar 2025 21:34:09 +0100 Subject: [PATCH 31/58] Fix schedules region filtering --- src/Appwrite/Migration/Version/V19.php | 3 +-- src/Appwrite/Platform/Tasks/ScheduleBase.php | 9 ++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 18234ebdc4..7adf5b85be 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -10,7 +10,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; use Utopia\Database\Query; -use Utopia\System\System; class V19 extends Migration { @@ -731,7 +730,7 @@ class V19 extends Migration if (empty($document->getAttribute('scheduleId', null))) { $schedule = $this->consoleDB->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), // Todo replace with projects region 'resourceType' => 'function', 'resourceId' => $document->getId(), 'resourceInternalId' => $document->getInternalId(), diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index dad2db0d9a..324b88d07b 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -103,8 +103,15 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); From f25143b240177b141c487048d5e9de03a85e7029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 25 Mar 2025 21:52:02 +0100 Subject: [PATCH 32/58] Support fra-region on cloud; temporary fix --- src/Appwrite/Migration/Version/V19.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 9 ++++++++- src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 7adf5b85be..4415003bfd 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -730,7 +730,7 @@ class V19 extends Migration if (empty($document->getAttribute('scheduleId', null))) { $schedule = $this->consoleDB->createDocument('schedules', new Document([ - 'region' => $project->getAttribute('region'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), 'resourceType' => 'function', 'resourceId' => $document->getId(), 'resourceInternalId' => $document->getInternalId(), diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 324b88d07b..a3c36cb96e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -160,8 +160,15 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6dff127784..8eaa08ce65 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -180,10 +180,17 @@ class Deletes extends Action */ private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void { + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From a461d5bf7281841ccbbe1c4a27d31e850b8dc2ad Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 26 Mar 2025 09:20:44 +0200 Subject: [PATCH 33/58] Deletes fix selects --- src/Appwrite/Platform/Workers/Deletes.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8eaa08ce65..2cc1eaf9af 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -31,6 +31,8 @@ use Utopia\System\System; class Deletes extends Action { + protected array $selects = ['$internalId', '$id', '$collection', '$updatedAt']; + public static function getName(): string { return 'deletes'; @@ -366,7 +368,7 @@ class Deletes extends Action $queries[] = Query::equal('resourceType', [$resourceType]); } - $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); + $queries[] = Query::select($this->selects); $queries[] = Query::orderAsc(); $this->deleteByGroup( @@ -403,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,7 +441,7 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -452,7 +454,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -749,7 +751,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -770,7 +772,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -807,7 +809,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), @@ -889,7 +891,7 @@ class Deletes extends Action */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('functionInternalId', [$functionInternalId]), Query::orderAsc() ], $dbForProject); From 424a3575034c31b34b6c2c80a24fb953338642bd Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 26 Mar 2025 10:02:30 +0200 Subject: [PATCH 34/58] Add $permissions --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 2cc1eaf9af..5033822241 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -31,7 +31,7 @@ use Utopia\System\System; class Deletes extends Action { - protected array $selects = ['$internalId', '$id', '$collection', '$updatedAt']; + protected array $selects = ['$internalId', '$id', '$collection', '$permissions', '$updatedAt']; public static function getName(): string { From d9d38197ddcd91c6429843ba5878b9a9a4e3bef7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 23:47:59 +1300 Subject: [PATCH 35/58] Update libs --- composer.json | 2 +- composer.lock | 65 ++++++++++++++++++++++----------------------------- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/composer.json b/composer.json index 5b0f26b883..b8c5afa109 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-multi-tenant-insert as 0.62.1", + "utopia-php/database": "0.63.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 09b8250af6..d4956aeba0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "640d2a8be42ae521ea12d8a908873f2b", + "content-hash": "5b3c46863e4571c838c30090ad96436c", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-multi-tenant-insert", + "version": "0.63.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0" + "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/2e01c999153a202a8bae9c21e4302e7a674442d0", - "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0", + "url": "https://api.github.com/repos/utopia-php/database/zipball/51ff0a6514e6eda44a6da016d838e7e55282e01a", + "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-multi-tenant-insert" + "source": "https://github.com/utopia-php/database/tree/0.63.0" }, - "time": "2025-03-25T04:45:25+00:00" + "time": "2025-03-26T10:35:37+00:00" }, { "name": "utopia-php/domains", @@ -3950,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.2", + "version": "0.8.3", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1" + "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", - "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", "shasum": "" }, "require": { @@ -3967,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.62.*", + "utopia-php/database": "0.63.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.2" + "source": "https://github.com/utopia-php/migration/tree/0.8.3" }, - "time": "2025-03-24T09:05:31+00:00" + "time": "2025-03-26T10:45:51+00:00" }, { "name": "utopia-php/orchestration", @@ -4385,16 +4385,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.8.2", + "version": "0.8.3", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4" + "reference": "1af73dd3e73987cf729c7db399054e4a70befd99" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4", - "reference": "5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/1af73dd3e73987cf729c7db399054e4a70befd99", + "reference": "1af73dd3e73987cf729c7db399054e4a70befd99", "shasum": "" }, "require": { @@ -4430,9 +4430,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.8.2" + "source": "https://github.com/utopia-php/swoole/tree/0.8.3" }, - "time": "2024-02-01T14:54:12+00:00" + "time": "2025-03-26T10:09:05+00:00" }, { "name": "utopia-php/system", @@ -4775,16 +4775,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.9", + "version": "0.40.10", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4" + "reference": "054ac96285caf4f77879087b2416a5ddb8263051" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/dbb45a5db22cdc3368fe2573c07ba6088f188fa4", - "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", + "reference": "054ac96285caf4f77879087b2416a5ddb8263051", "shasum": "" }, "require": { @@ -4820,9 +4820,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.9" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" }, - "time": "2025-03-17T18:39:14+00:00" + "time": "2025-03-25T13:44:16+00:00" }, { "name": "doctrine/annotations", @@ -8132,18 +8132,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-multi-tenant-insert", - "alias": "0.62.1", - "alias_normalized": "0.62.1.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From c79e8e5b954ce3e61c39bcd01413d05f7905b2dd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Mar 2025 18:04:17 +1300 Subject: [PATCH 36/58] Update database --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index d4956aeba0..eb29c765a0 100644 --- a/composer.lock +++ b/composer.lock @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.1", + "version": "v4.30.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/a4c4d8565b40b9f76debc9dfeb221412eacb8ced", + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.2" }, - "time": "2025-03-13T21:08:17+00:00" + "time": "2025-03-26T18:01:50+00:00" }, { "name": "league/csv", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.63.0", + "version": "0.63.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a" + "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/51ff0a6514e6eda44a6da016d838e7e55282e01a", - "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/ad191bf34151815f716f553796a363ff2b6ef7d3", + "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.0" + "source": "https://github.com/utopia-php/database/tree/0.63.1" }, - "time": "2025-03-26T10:35:37+00:00" + "time": "2025-03-27T04:58:07+00:00" }, { "name": "utopia-php/domains", From 17dc8e3f8ad82afc911e492b0c4be9218f0305a3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Mar 2025 21:37:10 +1300 Subject: [PATCH 37/58] Revert "chore(audits): return queue pre-fetch results" --- src/Appwrite/Platform/Workers/Audits.php | 49 +++++++++++------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 962bc622d1..ed5ff8010a 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,12 +12,12 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; -use Utopia\Queue\Result\Commit; -use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { + protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,7 +27,9 @@ class Audits extends Action protected function getBatchSize(): int { - return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; } public static function getName(): string @@ -55,13 +57,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return Commit|NoCommit + * @return void * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit + public function action(Message $message, callable $getProjectDB, Document $project): void { $payload = $message->getPayload() ?? []; @@ -117,34 +119,29 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs), 0); - $shouldProcessBatch = $logCount >= $batchSize; - if (!$shouldProcessBatch && $logCount > 0) { + $shouldProcessBatch = \count($this->logs) >= $batchSize; + if (!$shouldProcessBatch && \count($this->logs) > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if (!$shouldProcessBatch) { - return new NoCommit(); - } + if ($shouldProcessBatch) { + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); - - unset($this->logs[$internalId]); + unset($this->logs[$internalId]); + } + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + $this->lastTriggeredTime = time(); } - return new Commit(); - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - return new NoCommit(); - } finally { - $this->lastTriggeredTime = time(); } } } From cba700402bb79598a526f0e6d026060cf0cd3e50 Mon Sep 17 00:00:00 2001 From: fogelito Date: Fri, 28 Mar 2025 09:04:50 +0300 Subject: [PATCH 38/58] Add order by for cursor --- src/Appwrite/Platform/Workers/Deletes.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 5033822241..5674448c18 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -405,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['accessedAt'])), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,9 +439,11 @@ class Deletes extends Action /** @var Database $dbForProject*/ $dbForProject = $getProjectDB($project); + $select = array_merge($this->selects, ['time']); + // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select($this->selects), + Query::select($select), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -454,7 +456,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select($this->selects), + Query::select($select), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -751,7 +753,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['$createdAt'])), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -772,7 +774,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['$createdAt'])), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -809,7 +811,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['time'])), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), From a4c18c4b774eefa25af547b31929986d953a2552 Mon Sep 17 00:00:00 2001 From: fogelito Date: Fri, 28 Mar 2025 09:16:25 +0300 Subject: [PATCH 39/58] Remove array merge --- src/Appwrite/Platform/Workers/Deletes.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 5674448c18..a9b83976a4 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -405,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select(array_merge($this->selects, ['accessedAt'])), + Query::select([...$this->selects, 'accessedAt']), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,11 +439,11 @@ class Deletes extends Action /** @var Database $dbForProject*/ $dbForProject = $getProjectDB($project); - $select = array_merge($this->selects, ['time']); + $selects = [...$this->selects, 'time']; // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select($select), + Query::select($selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -456,7 +456,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select($select), + Query::select($selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -753,7 +753,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select(array_merge($this->selects, ['$createdAt'])), + Query::select([...$this->selects, '$createdAt']), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -774,7 +774,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select(array_merge($this->selects, ['$createdAt'])), + Query::select([...$this->selects, '$createdAt']), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -811,7 +811,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select(array_merge($this->selects, ['time'])), + Query::select([...$this->selects, 'time']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), From 22939831f2d759f26bc5912b2b3e7a9cdceadc62 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 28 Mar 2025 20:58:25 +0100 Subject: [PATCH 40/58] Updated fetch version --- composer.json | 2 +- composer.lock | 78 +++++++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/composer.json b/composer.json index b8c5afa109..176b5889fc 100644 --- a/composer.json +++ b/composer.json @@ -55,7 +55,7 @@ "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", - "utopia-php/fetch": "0.3.*", + "utopia-php/fetch": "0.4.*", "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", diff --git a/composer.lock b/composer.lock index eb29c765a0..3772b25707 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5b3c46863e4571c838c30090ad96436c", + "content-hash": "c99b4733669c17013e211c7dc54a86f6", "packages": [ { "name": "adhocore/jwt", @@ -3659,16 +3659,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.3.1", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0" + "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", - "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/46e791ff6a95864517750b9df6bbf4a17e3c9c4e", + "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e", "shasum": "" }, "require": { @@ -3692,9 +3692,9 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.3.1" + "source": "https://github.com/utopia-php/fetch/tree/0.4.0" }, - "time": "2025-03-05T18:08:55+00:00" + "time": "2025-03-11T21:06:56+00:00" }, { "name": "utopia-php/framework", @@ -3950,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.3", + "version": "0.8.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/845fd04ccf5e0edb03c184b864e0596080a432b8", + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8", "shasum": "" }, "require": { @@ -3967,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.63.*", + "utopia-php/database": "0.*.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.3" + "source": "https://github.com/utopia-php/migration/tree/0.8.4" }, - "time": "2025-03-26T10:45:51+00:00" + "time": "2025-03-28T02:08:22+00:00" }, { "name": "utopia-php/orchestration", @@ -4211,23 +4211,23 @@ }, { "name": "utopia-php/queue", - "version": "0.9.0", + "version": "0.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7" + "reference": "32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/077075f1d57afa430f76c35ed3bf4616e0eee8e7", - "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32", + "reference": "32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32", "shasum": "" }, "require": { "php": ">=8.3", "php-amqplib/php-amqplib": "^3.7", "utopia-php/cli": "0.15.*", - "utopia-php/fetch": "^0.3.0", + "utopia-php/fetch": "0.4.*", "utopia-php/framework": "0.33.*", "utopia-php/telemetry": "0.1.*" }, @@ -4270,9 +4270,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.9.0" + "source": "https://github.com/utopia-php/queue/tree/0.9.1" }, - "time": "2025-03-13T12:22:41+00:00" + "time": "2025-03-28T19:49:36+00:00" }, { "name": "utopia-php/registry", @@ -4775,16 +4775,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.10", + "version": "0.40.11", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051" + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", "shasum": "" }, "require": { @@ -4820,9 +4820,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" }, - "time": "2025-03-25T13:44:16+00:00" + "time": "2025-03-26T10:53:16+00:00" }, { "name": "doctrine/annotations", @@ -7154,16 +7154,16 @@ }, { "name": "symfony/console", - "version": "v7.2.1", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" + "reference": "e51498ea18570c062e7df29d05a7003585b19b88" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "url": "https://api.github.com/repos/symfony/console/zipball/e51498ea18570c062e7df29d05a7003585b19b88", + "reference": "e51498ea18570c062e7df29d05a7003585b19b88", "shasum": "" }, "require": { @@ -7227,7 +7227,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.1" + "source": "https://github.com/symfony/console/tree/v7.2.5" }, "funding": [ { @@ -7243,7 +7243,7 @@ "type": "tidelift" } ], - "time": "2024-12-11T03:49:26+00:00" + "time": "2025-03-12T08:11:12+00:00" }, { "name": "symfony/filesystem", @@ -7758,16 +7758,16 @@ }, { "name": "symfony/process", - "version": "v7.2.4", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf" + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", + "url": "https://api.github.com/repos/symfony/process/zipball/87b7c93e57df9d8e39a093d32587702380ff045d", + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d", "shasum": "" }, "require": { @@ -7799,7 +7799,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.2.4" + "source": "https://github.com/symfony/process/tree/v7.2.5" }, "funding": [ { @@ -7815,7 +7815,7 @@ "type": "tidelift" } ], - "time": "2025-02-05T08:33:46+00:00" + "time": "2025-03-13T12:21:46+00:00" }, { "name": "symfony/string", From 612b064d2a9121949030fe7459ad43732a36e15c Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 31 Mar 2025 15:16:59 +0300 Subject: [PATCH 41/58] 0.63.2 --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 176b5889fc..2e25ca3748 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.63.*", + "utopia-php/database": "dev-0.63.2-fix as 0.63.2", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", From 4698323f38cfd0f49040eac13da1c63b8cca6003 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 31 Mar 2025 16:19:17 +0300 Subject: [PATCH 42/58] lock --- composer.lock | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/composer.lock b/composer.lock index 3772b25707..a25995668e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c99b4733669c17013e211c7dc54a86f6", + "content-hash": "796b9a9952aac4604cf2acfd0ce455c6", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.63.1", + "version": "dev-0.63.2-fix", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3" + "reference": "f32c48a8b94aa2e4b543947046daf59e7cbd49d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/ad191bf34151815f716f553796a363ff2b6ef7d3", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f32c48a8b94aa2e4b543947046daf59e7cbd49d4", + "reference": "f32c48a8b94aa2e4b543947046daf59e7cbd49d4", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.1" + "source": "https://github.com/utopia-php/database/tree/0.63.2-fix" }, - "time": "2025-03-27T04:58:07+00:00" + "time": "2025-03-31T11:53:46+00:00" }, { "name": "utopia-php/domains", @@ -8132,9 +8132,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-0.63.2-fix", + "alias": "0.63.2", + "alias_normalized": "0.63.2.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From d92413fd6ba46eb79f548aa957e4258a8d14f7f7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 1 Apr 2025 08:13:56 +0300 Subject: [PATCH 43/58] Revert database lib --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2e25ca3748..176b5889fc 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-0.63.2-fix as 0.63.2", + "utopia-php/database": "0.63.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", From b46de29735db1de51981aff820440eddad4bbbfe Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 1 Apr 2025 08:15:03 +0300 Subject: [PATCH 44/58] database 0.63.2 --- composer.lock | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/composer.lock b/composer.lock index a25995668e..06aea5ed69 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "796b9a9952aac4604cf2acfd0ce455c6", + "content-hash": "c99b4733669c17013e211c7dc54a86f6", "packages": [ { "name": "adhocore/jwt", @@ -3497,7 +3497,7 @@ }, { "name": "utopia-php/database", - "version": "dev-0.63.2-fix", + "version": "0.63.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", @@ -3546,7 +3546,7 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.2-fix" + "source": "https://github.com/utopia-php/database/tree/0.63.2" }, "time": "2025-03-31T11:53:46+00:00" }, @@ -8132,18 +8132,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-0.63.2-fix", - "alias": "0.63.2", - "alias_normalized": "0.63.2.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 938505da5121dc08c84db9ee39b36d332133fe62 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 1 Apr 2025 21:23:23 +0200 Subject: [PATCH 45/58] Updated websocket version --- composer.json | 2 +- composer.lock | 33 ++++++++++++--------------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/composer.json b/composer.json index 176b5889fc..39ac2f6574 100644 --- a/composer.json +++ b/composer.json @@ -72,7 +72,7 @@ "utopia-php/system": "0.9.*", "utopia-php/telemetry": "0.1.*", "utopia-php/vcs": "0.9.*", - "utopia-php/websocket": "0.1.*", + "utopia-php/websocket": "0.3.*", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", "phpmailer/phpmailer": "6.9.1", diff --git a/composer.lock b/composer.lock index 06aea5ed69..8ee7cd75f0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c99b4733669c17013e211c7dc54a86f6", + "content-hash": "d393dfb19f208bab8b2f165071915fc9", "packages": [ { "name": "adhocore/jwt", @@ -4592,27 +4592,28 @@ }, { "name": "utopia-php/websocket", - "version": "0.1.0", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/websocket.git", - "reference": "51fcb86171400d8aa40d76c54593481fd273dab5" + "reference": "629e53640b108eab43c7cc9ab375efade8622d43" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/websocket/zipball/51fcb86171400d8aa40d76c54593481fd273dab5", - "reference": "51fcb86171400d8aa40d76c54593481fd273dab5", + "url": "https://api.github.com/repos/utopia-php/websocket/zipball/629e53640b108eab43c7cc9ab375efade8622d43", + "reference": "629e53640b108eab43c7cc9ab375efade8622d43", "shasum": "" }, "require": { "php": ">=8.0" }, "require-dev": { + "laravel/pint": "^1.15", + "phpstan/phpstan": "^1.12", "phpunit/phpunit": "^9.5.5", - "swoole/ide-helper": "4.6.6", + "swoole/ide-helper": "5.1.2", "textalk/websocket": "1.5.2", - "vimeo/psalm": "^4.8.1", - "workerman/workerman": "^4.0" + "workerman/workerman": "4.1.*" }, "type": "library", "autoload": { @@ -4624,16 +4625,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Torsten Dittmann", - "email": "torsten@appwrite.io" - } - ], "description": "A simple abstraction for WebSocket servers.", "keywords": [ "framework", @@ -4644,9 +4635,9 @@ ], "support": { "issues": "https://github.com/utopia-php/websocket/issues", - "source": "https://github.com/utopia-php/websocket/tree/0.1.0" + "source": "https://github.com/utopia-php/websocket/tree/0.3.0" }, - "time": "2021-12-20T10:50:09+00:00" + "time": "2025-03-28T01:11:13+00:00" }, { "name": "webmozart/assert", @@ -8134,7 +8125,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From c152d8e001a4b44e9c4069e6fdfb99f96a9a7077 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 2 Apr 2025 13:02:35 +1300 Subject: [PATCH 46/58] Update database --- composer.json | 4 +-- composer.lock | 92 ++++++++++++++++++++++++++++----------------------- 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/composer.json b/composer.json index b8c5afa109..2d13ef244f 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.63.*", + "utopia-php/database": "dev-fix-tenant as 0.64.0", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", @@ -63,7 +63,7 @@ "utopia-php/migration": "0.8.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.7.*", + "utopia-php/pools": "0.8.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.9.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index eb29c765a0..43fb23d14f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5b3c46863e4571c838c30090ad96436c", + "content-hash": "0f7e98ef962fcce12b0ea435188504aa", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.63.1", + "version": "dev-fix-tenant", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3" + "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/ad191bf34151815f716f553796a363ff2b6ef7d3", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3", + "url": "https://api.github.com/repos/utopia-php/database/zipball/1b7d8b774ae28d98de82a86e4fef262514e8fc40", + "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40", "shasum": "" }, "require": { @@ -3514,7 +3514,8 @@ "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.12.*", - "utopia-php/framework": "0.33.*" + "utopia-php/framework": "0.33.*", + "utopia-php/pools": "0.8.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -3546,9 +3547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.1" + "source": "https://github.com/utopia-php/database/tree/fix-tenant" }, - "time": "2025-03-27T04:58:07+00:00" + "time": "2025-04-02T00:00:03+00:00" }, { "name": "utopia-php/domains", @@ -3950,16 +3951,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.3", + "version": "0.8.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/845fd04ccf5e0edb03c184b864e0596080a432b8", + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8", "shasum": "" }, "require": { @@ -3967,7 +3968,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.63.*", + "utopia-php/database": "0.*.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4001,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.3" + "source": "https://github.com/utopia-php/migration/tree/0.8.4" }, - "time": "2025-03-26T10:45:51+00:00" + "time": "2025-03-28T02:08:22+00:00" }, { "name": "utopia-php/orchestration", @@ -4106,16 +4107,16 @@ }, { "name": "utopia-php/pools", - "version": "0.7.0", + "version": "0.8.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf" + "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/ad64d45afda08ec8b29e2642a8d18075964d40bf", - "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/60733929dc328e7ea47e800579c8bbf0d49df5ba", + "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba", "shasum": "" }, "require": { @@ -4152,9 +4153,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.7.0" + "source": "https://github.com/utopia-php/pools/tree/0.8.0" }, - "time": "2025-03-18T03:55:33+00:00" + "time": "2025-03-19T10:22:03+00:00" }, { "name": "utopia-php/preloader", @@ -4775,16 +4776,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.10", + "version": "0.40.11", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051" + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", "shasum": "" }, "require": { @@ -4820,9 +4821,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" }, - "time": "2025-03-25T13:44:16+00:00" + "time": "2025-03-26T10:53:16+00:00" }, { "name": "doctrine/annotations", @@ -7154,16 +7155,16 @@ }, { "name": "symfony/console", - "version": "v7.2.1", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" + "reference": "e51498ea18570c062e7df29d05a7003585b19b88" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "url": "https://api.github.com/repos/symfony/console/zipball/e51498ea18570c062e7df29d05a7003585b19b88", + "reference": "e51498ea18570c062e7df29d05a7003585b19b88", "shasum": "" }, "require": { @@ -7227,7 +7228,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.1" + "source": "https://github.com/symfony/console/tree/v7.2.5" }, "funding": [ { @@ -7243,7 +7244,7 @@ "type": "tidelift" } ], - "time": "2024-12-11T03:49:26+00:00" + "time": "2025-03-12T08:11:12+00:00" }, { "name": "symfony/filesystem", @@ -7758,16 +7759,16 @@ }, { "name": "symfony/process", - "version": "v7.2.4", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf" + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", + "url": "https://api.github.com/repos/symfony/process/zipball/87b7c93e57df9d8e39a093d32587702380ff045d", + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d", "shasum": "" }, "require": { @@ -7799,7 +7800,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.2.4" + "source": "https://github.com/symfony/process/tree/v7.2.5" }, "funding": [ { @@ -7815,7 +7816,7 @@ "type": "tidelift" } ], - "time": "2025-02-05T08:33:46+00:00" + "time": "2025-03-13T12:21:46+00:00" }, { "name": "symfony/string", @@ -8132,9 +8133,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-fix-tenant", + "alias": "0.64.0", + "alias_normalized": "0.64.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 32a5ff4c8c5e42c9ad0652f20cb91764089e31e8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 2 Apr 2025 13:37:26 +1300 Subject: [PATCH 47/58] Update release --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index 0eb0f58f04..3920351c06 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-fix-tenant as 0.64.0", + "utopia-php/database": "0.64.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 80d9c144f5..270fbca6a4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d9bc106a753e57907134bb717d486310", + "content-hash": "6a54c8bc4f9f14cd3883f55880864630", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "dev-fix-tenant", + "version": "0.64.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40" + "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/1b7d8b774ae28d98de82a86e4fef262514e8fc40", - "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40", + "url": "https://api.github.com/repos/utopia-php/database/zipball/6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", + "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", "shasum": "" }, "require": { @@ -3547,9 +3547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/fix-tenant" + "source": "https://github.com/utopia-php/database/tree/0.64.1" }, - "time": "2025-04-02T00:00:03+00:00" + "time": "2025-04-02T00:35:29+00:00" }, { "name": "utopia-php/domains", @@ -8124,18 +8124,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-fix-tenant", - "alias": "0.64.0", - "alias_normalized": "0.64.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 1660c0b00c15c216d1f422719217893fc92349cd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 3 Apr 2025 02:44:29 +0000 Subject: [PATCH 48/58] Fix: image transformation cache --- app/controllers/shared/api.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 1015400a12..87382bbfe2 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -390,7 +390,8 @@ App::init() ->inject('timelimit') ->inject('mode') ->inject('apiKey') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode, ?Key $apiKey) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('plan') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode, ?Key $apiKey, array $plan) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -520,6 +521,10 @@ App::init() $useCache = $route->getLabel('cache', false); if ($useCache) { + $route = $utopia->match($request); + $isImageTransformation = $route->getPath() === '/v1/storage/buckets/:bucketId/files/:fileId/preview'; + $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && !Auth::isPrivilegedUser(Authorization::getRoles()); + $key = md5($request->getURI() . '*' . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( @@ -532,7 +537,7 @@ App::init() $parts = explode('/', $cacheLog->getAttribute('resourceType')); $type = $parts[0] ?? null; - if ($type === 'bucket') { + if ($type === 'bucket' && (!$isImageTransformation || !$isDisabled)) { $bucketId = $parts[1] ?? null; $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -573,8 +578,10 @@ App::init() $response ->addHeader('Cache-Control', sprintf('private, max-age=%d', $timestamp)) ->addHeader('X-Appwrite-Cache', 'hit') - ->setContentType($cacheLog->getAttribute('mimeType')) - ->send($data); + ->setContentType($cacheLog->getAttribute('mimeType')); + if (!$isImageTransformation || !$isDisabled) { + $response->send($data); + } } else { $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') From bd87672607cbeb8c3152e8f3878d059bafb1daa9 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 3 Apr 2025 18:31:50 +0300 Subject: [PATCH 49/58] Remove audit payload --- src/Appwrite/Platform/Workers/Audits.php | 6 +++++- src/Appwrite/Platform/Workers/Databases.php | 3 ++- src/Appwrite/Platform/Workers/Deletes.php | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index ed5ff8010a..76309145b8 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -74,7 +74,11 @@ class Audits extends Action Console::info('Aggregating audit logs'); $event = $payload['event'] ?? ''; - $auditPayload = $payload['payload'] ?? ''; + + $auditPayload = ''; + if ($project->getId() === 'console') { + $auditPayload = $payload['payload'] ?? ''; + } $mode = $payload['mode'] ?? ''; $resource = $payload['resource'] ?? ''; $userAgent = $payload['userAgent'] ?? ''; diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 4abd035599..44738b557c 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -565,7 +565,8 @@ class Databases extends Action try { $documents = $database->deleteDocuments($collectionId, $queries); } catch (\Throwable $th) { - Console::error('Failed to delete documents for collection ' . $collectionId . ': ' . $th->getMessage()); + $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; + Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collectionId} {$tenant} :{$th->getMessage()}"); return; } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index a9b83976a4..09c60c7740 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1050,7 +1050,8 @@ class Deletes extends Action try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { - Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage()); + $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; + Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}"); return; } From 2a672f5a633b98f1bcecc73d3addbfc78eecc4b5 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 5 Apr 2025 19:20:24 +0530 Subject: [PATCH 50/58] fix: don't allow the only owner in an org to change their role to anything else. --- app/controllers/api/teams.php | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index dfdccf5e99..d4db6a4a52 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1034,7 +1034,6 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') ->param('membershipId', '', new UID(), 'Membership ID.') ->param('roles', [], function (Document $project) { if ($project->getId() === 'console') { - ; $roles = array_keys(Config::getParam('roles', [])); array_filter($roles, function ($role) { return !in_array($role, [Auth::USER_ROLE_APPS, Auth::USER_ROLE_GUESTS, Auth::USER_ROLE_USERS]); @@ -1046,9 +1045,10 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') ->inject('request') ->inject('response') ->inject('user') + ->inject('project') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $teamId, string $membershipId, array $roles, Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { + ->action(function (string $teamId, string $membershipId, array $roles, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) { $team = $dbForProject->getDocument('teams', $teamId); if ($team->isEmpty()) { @@ -1069,6 +1069,21 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') $isAppUser = Auth::isAppUser(Authorization::getRoles()); $isOwner = Authorization::isRole('team:' . $team->getId() . '/owner'); + if ($project->getId() === 'console') { + // Quick check: fetch up to 2 owners to determine if only one exists + $ownersCount = $dbForProject->count( + collection: 'memberships', + queries: [Query::contains('roles', ['owner'])], + max: 2 + ); + + // If there's only one owner, + // and the requester is that owner, prevent role change + if ($ownersCount === 1 && $isOwner) { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'There must be at least one owner in the organization.'); + } + } + if (!$isOwner && !$isPrivilegedUser && !$isAppUser) { // Not owner, not admin, not app (server) throw new Exception(Exception::USER_UNAUTHORIZED, 'User is not allowed to modify roles'); } From 7297188c1bf0dd67229afc2ef30ad527cc76fecc Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 5 Apr 2025 19:42:20 +0530 Subject: [PATCH 51/58] update: roles check. --- app/controllers/api/teams.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index d4db6a4a52..3e0e366b6b 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1077,9 +1077,9 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') max: 2 ); - // If there's only one owner, - // and the requester is that owner, prevent role change - if ($ownersCount === 1 && $isOwner) { + // Prevent role change if there's only one owner left, + // the requester is that owner, and the new `$roles` no longer include 'owner'! + if ($ownersCount === 1 && $isOwner && !\in_array('owner', $roles)) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'There must be at least one owner in the organization.'); } } From abd953974d4dec5af095ced36dec79ab0904df5f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 5 Apr 2025 14:43:07 +0000 Subject: [PATCH 52/58] chore: give read write permission to vendor for gitpod config --- .gitpod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index e9c0928229..c103d81900 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,13 +1,14 @@ tasks: - name: Run Appwrite Docker Stack init: | - git config --global --add safe.directory /app git submodule update --init docker compose build docker compose pull docker pull composer command: | + sudo chown -R $(id -u):$(id -g) vendor/ docker run --rm --interactive --tty \ + --user "$(id -u):$(id -g)" \ --volume $PWD:/app \ composer install \ --ignore-platform-reqs \ From e5ef825ac41fd63fad3983115730c48a3725fd14 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 5 Apr 2025 15:47:22 +0000 Subject: [PATCH 53/58] chore: update php image to 0.8.1 --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index 270fbca6a4..8db4706bb5 100644 --- a/composer.lock +++ b/composer.lock @@ -2965,16 +2965,16 @@ }, { "name": "tbachert/spi", - "version": "v1.0.2", + "version": "v1.0.3", "source": { "type": "git", "url": "https://github.com/Nevay/spi.git", - "reference": "2ddfaf815dafb45791a61b08170de8d583c16062" + "reference": "506a79c98e1a51522e76ee921ccb6c62d52faf3a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Nevay/spi/zipball/2ddfaf815dafb45791a61b08170de8d583c16062", - "reference": "2ddfaf815dafb45791a61b08170de8d583c16062", + "url": "https://api.github.com/repos/Nevay/spi/zipball/506a79c98e1a51522e76ee921ccb6c62d52faf3a", + "reference": "506a79c98e1a51522e76ee921ccb6c62d52faf3a", "shasum": "" }, "require": { @@ -3011,9 +3011,9 @@ ], "support": { "issues": "https://github.com/Nevay/spi/issues", - "source": "https://github.com/Nevay/spi/tree/v1.0.2" + "source": "https://github.com/Nevay/spi/tree/v1.0.3" }, - "time": "2024-10-04T16:36:12+00:00" + "time": "2025-04-02T19:38:14+00:00" }, { "name": "thecodingmachine/safe", @@ -3746,16 +3746,16 @@ }, { "name": "utopia-php/image", - "version": "0.8.0", + "version": "0.8.1", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "dcae5b1c6deb3ff6865f4e68f012b3709c289bca" + "reference": "e8cc7dd14f423270a1b7570ec0dae88a66195b63" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/dcae5b1c6deb3ff6865f4e68f012b3709c289bca", - "reference": "dcae5b1c6deb3ff6865f4e68f012b3709c289bca", + "url": "https://api.github.com/repos/utopia-php/image/zipball/e8cc7dd14f423270a1b7570ec0dae88a66195b63", + "reference": "e8cc7dd14f423270a1b7570ec0dae88a66195b63", "shasum": "" }, "require": { @@ -3789,9 +3789,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.8.0" + "source": "https://github.com/utopia-php/image/tree/0.8.1" }, - "time": "2025-02-20T11:49:03+00:00" + "time": "2025-04-04T18:55:20+00:00" }, { "name": "utopia-php/locale", From 403083b0cef7053588474b98587b036790fdc01c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 6 Apr 2025 08:31:46 +0530 Subject: [PATCH 54/58] Update .gitpod.yml --- .gitpod.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index c103d81900..78375bb1f0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -6,7 +6,6 @@ tasks: docker compose pull docker pull composer command: | - sudo chown -R $(id -u):$(id -g) vendor/ docker run --rm --interactive --tty \ --user "$(id -u):$(id -g)" \ --volume $PWD:/app \ From c3c568653fd9f4a5693e5b06a8c4d6c342e30fc6 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sun, 6 Apr 2025 10:21:55 +0530 Subject: [PATCH 55/58] add: roles check test. --- tests/e2e/Services/Teams/TeamsBase.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index 2328e4cdbf..87486696d2 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -37,6 +37,31 @@ trait TeamsBase $teamUid = $response1['body']['$id']; $teamName = $response1['body']['name']; + /** + * Test: Attempt to downgrade the only OWNER in a team (should fail) + */ + // Step 1: Fetch all team memberships — only one exists at this point + $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + // Step 2: Extract the membership ID of the only member (also the only OWNER) + $membershipID = $response['body']['memberships'][0]['$id']; + + // Step 3: Attempt to downgrade the member's role to 'developer' + $response = $this->client->call(Client::METHOD_PATCH, '/teams/' . $teamUid . '/memberships/' . $membershipID, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'roles' => ['developer'] + ]); + + // Step 4: Assert failure — cannot remove the only OWNER from a team + $this->assertEquals(400, $response['headers']['status-code']); + $this->assertEquals('general_argument_invalid', $response['body']['type']); + $this->assertEquals('There must be at least one owner in the organization.', $response['body']['message']); + $teamId = ID::unique(); $response2 = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ 'content-type' => 'application/json', From 2243f0c72e45e5a8abfe716bb2b8470375a07f6b Mon Sep 17 00:00:00 2001 From: Darshan Date: Sun, 6 Apr 2025 10:29:18 +0530 Subject: [PATCH 56/58] fix: test for only console db <> orgs. --- tests/e2e/Services/Teams/TeamsBase.php | 44 +++++++++++++++----------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index 87486696d2..80ac1621ee 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -38,29 +38,35 @@ trait TeamsBase $teamName = $response1['body']['name']; /** - * Test: Attempt to downgrade the only OWNER in a team (should fail) + * Test: Attempt to downgrade the only OWNER in an organization (should fail) */ - // Step 1: Fetch all team memberships — only one exists at this point - $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + if ($this->getProject()['$id'] === 'console') { + // Step 1: Fetch all team memberships — only one exists at this point + $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::limit(1)->toString(), + ], + ]); - // Step 2: Extract the membership ID of the only member (also the only OWNER) - $membershipID = $response['body']['memberships'][0]['$id']; + // Step 2: Extract the membership ID of the only member (also the only OWNER) + $membershipID = $response['body']['memberships'][0]['$id']; - // Step 3: Attempt to downgrade the member's role to 'developer' - $response = $this->client->call(Client::METHOD_PATCH, '/teams/' . $teamUid . '/memberships/' . $membershipID, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'roles' => ['developer'] - ]); + // Step 3: Attempt to downgrade the member's role to 'developer' + $response = $this->client->call(Client::METHOD_PATCH, '/teams/' . $teamUid . '/memberships/' . $membershipID, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'roles' => ['developer'] + ]); - // Step 4: Assert failure — cannot remove the only OWNER from a team - $this->assertEquals(400, $response['headers']['status-code']); - $this->assertEquals('general_argument_invalid', $response['body']['type']); - $this->assertEquals('There must be at least one owner in the organization.', $response['body']['message']); + // Step 4: Assert failure — cannot remove the only OWNER from a team + $this->assertEquals(400, $response['headers']['status-code']); + $this->assertEquals('general_argument_invalid', $response['body']['type']); + $this->assertEquals('There must be at least one owner in the organization.', $response['body']['message']); + } $teamId = ID::unique(); $response2 = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ From 954ed45a36ff821125cbfa63c75f16aebc7fb8f1 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Fri, 28 Feb 2025 13:52:34 +0100 Subject: [PATCH 57/58] feat: update appwrite to use generic S3 adapter --- app/init/resources.php | 15 +++++++++++++-- docker-compose.yml | 6 +++++- src/Executor/Executor.php | 6 +++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index 4e53b24c06..0011a8c677 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -38,6 +38,7 @@ use Utopia\Logger\Log; use Utopia\Pools\Group; use Utopia\Queue\Publisher; use Utopia\Storage\Device; +use Utopia\Storage\Device\AWS; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; @@ -540,7 +541,12 @@ function getDevice(string $root, string $connection = ''): Device switch ($device) { case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl, $url); + if (!empty($url)) { + return new S3($root, $accessKey, $accessSecret, $url, $region, $acl); + } else { + return new AWS($root, $accessKey, $accessSecret, $bucket, $region, $acl); + } + // no break case STORAGE::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); @@ -567,7 +573,12 @@ function getDevice(string $root, string $connection = ''): Device $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); $s3Acl = 'private'; $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl, $s3EndpointUrl); + if (!empty($s3EndpointUrl)) { + return new S3($root, $s3AccessKey, $s3SecretKey, $s3EndpointUrl, $s3Region, $s3Acl); + } else { + return new AWS($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + } + // no break case Storage::DEVICE_DO_SPACES: $doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); $doSpacesSecretKey = System::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); diff --git a/docker-compose.yml b/docker-compose.yml index 05ddba967a..8c8a364f30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -198,6 +198,8 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_DATABASE_SHARED_NAMESPACE - _APP_FUNCTIONS_CREATION_ABUSE_LIMIT + extra_hosts: + - "host.docker.internal:host-gateway" appwrite-console: <<: *x-logging @@ -489,6 +491,8 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES + extra_hosts: + - "host.docker.internal:host-gateway" appwrite-worker-certificates: entrypoint: worker-certificates @@ -1131,4 +1135,4 @@ volumes: appwrite-certificates: appwrite-functions: appwrite-builds: - appwrite-config: \ No newline at end of file + appwrite-config: diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index c230cfb664..936fba5b24 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -235,7 +235,11 @@ class Executor throw new \Exception($message, $status); } - $response['body']['headers'] = \json_decode($response['body']['headers'] ?? '{}', true); + $headers = $response['body']['headers'] ?? []; + if (is_string($headers)) { + $headers = \json_decode($headers, true); + } + $response['body']['headers'] = $headers; $response['body']['statusCode'] = \intval($response['body']['statusCode'] ?? 500); $response['body']['duration'] = \floatval($response['body']['duration'] ?? 0); $response['body']['startTime'] = \floatval($response['body']['startTime'] ?? \microtime(true)); From 0ae70f8321e1916eaad4f6afa1cd430e55f7b3b9 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 25 Mar 2025 12:33:03 +0100 Subject: [PATCH 58/58] feat: inject executor instead of creating a new instance --- app/cli.php | 3 ++ app/controllers/api/functions.php | 12 +++---- app/controllers/general.php | 23 +++++++------ app/init/resources.php | 3 ++ app/worker.php | 3 ++ src/Appwrite/Platform/Workers/Builds.php | 15 +++++---- src/Appwrite/Platform/Workers/Deletes.php | 26 ++++++++------- src/Appwrite/Platform/Workers/Functions.php | 11 ++++-- src/Executor/Executor.php | 37 +++++++++++++-------- 9 files changed, 82 insertions(+), 51 deletions(-) diff --git a/app/cli.php b/app/cli.php index f080217365..82f229673e 100644 --- a/app/cli.php +++ b/app/cli.php @@ -9,6 +9,7 @@ use Appwrite\Event\StatsResources; use Appwrite\Event\StatsUsage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; +use Executor\Executor; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\CLI; @@ -255,6 +256,8 @@ CLI::setResource('logError', function (Registry $register) { }; }, ['register']); +CLI::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); + $platform = new Appwrite(); $platform->init(Service::TYPE_TASK); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 37932e4165..c7f24f984a 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -869,7 +869,8 @@ App::put('/v1/functions/:functionId') ->inject('queueForBuilds') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->inject('executor') + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github, Executor $executor) use ($redeployVcs) { // TODO: If only branch changes, re-deploy $function = $dbForProject->getDocument('functions', $functionId); @@ -972,7 +973,6 @@ App::put('/v1/functions/:functionId') // Enforce Cold Start if spec limits change. if ($function->getAttribute('specification') !== $specification && !empty($function->getAttribute('deployment'))) { - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { $executor->deleteRuntime($project->getId(), $function->getAttribute('deployment')); } catch (\Throwable $th) { @@ -1779,7 +1779,8 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId/build') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') - ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents) { + ->inject('executor') + ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Executor $executor) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1834,7 +1835,6 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId/build') $dbForProject->purgeCachedDocument('deployments', $deployment->getId()); try { - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); $executor->deleteRuntime($project->getId(), $deploymentId . "-build"); } catch (\Throwable $th) { // Don't throw if the deployment doesn't exist @@ -1886,8 +1886,9 @@ App::post('/v1/functions/:functionId/executions') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') - ->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb) { $async = \strval($async) === 'true' || \strval($async) === '1'; if (!$async && !is_null($scheduledAt)) { @@ -2160,7 +2161,6 @@ App::post('/v1/functions/:functionId/executions') ]); /** Execute function */ - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); try { $version = $function->getAttribute('version', 'v2'); $command = $runtime['startCommand']; diff --git a/app/controllers/general.php b/app/controllers/general.php index 1423763cec..6c1167cf76 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -50,7 +50,7 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) +function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); @@ -339,7 +339,6 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw ]); /** Execute function */ - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); try { $version = $function->getAttribute('version', 'v2'); $command = $runtime['startCommand']; @@ -503,9 +502,10 @@ App::init() ->inject('queueForEvents') ->inject('queueForCertificates') ->inject('queueForFunctions') + ->inject('executor') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, Executor $executor, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ @@ -513,7 +513,7 @@ App::init() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -742,11 +742,12 @@ App::options() ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') ->inject('project') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { /* * Appwrite Router */ @@ -754,7 +755,7 @@ App::options() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -1062,10 +1063,11 @@ App::get('/robots.txt') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -1073,7 +1075,7 @@ App::get('/robots.txt') $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); + router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname); } }); @@ -1090,10 +1092,11 @@ App::get('/humans.txt') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -1101,7 +1104,7 @@ App::get('/humans.txt') $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); + router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname); } }); diff --git a/app/init/resources.php b/app/init/resources.php index 0011a8c677..2360179913 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -21,6 +21,7 @@ use Appwrite\Extend\Exception; use Appwrite\GraphQL\Schema; use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Request; +use Executor\Executor; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -833,3 +834,5 @@ App::setResource('apiKey', function (Request $request, Document $project): ?Key return Key::decode($project, $key); }, ['request', 'project']); + +App::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); diff --git a/app/worker.php b/app/worker.php index 90496c0430..6a51ee55be 100644 --- a/app/worker.php +++ b/app/worker.php @@ -18,6 +18,7 @@ use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; +use Executor\Executor; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -413,6 +414,8 @@ Server::setResource('logError', function (Registry $register, Document $project) }; }, ['register', 'project']); +Server::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); + $pools = $register->get('pools'); $platform = new Appwrite(); $args = $platform->getEnv('argv'); diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 6f26e9a80c..4057d4b190 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -59,8 +59,9 @@ class Builds extends Action ->inject('deviceForFunctions') ->inject('isResourceBlocked') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => - $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); + ->inject('executor') + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor) => + $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log, $executor)); } /** @@ -76,10 +77,11 @@ class Builds extends Action * @param Database $dbForProject * @param Device $deviceForFunctions * @param Log $log + * @param Executor $executor * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor): void { $payload = $message->getPayload() ?? []; @@ -100,7 +102,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); + $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log, $executor); break; default: @@ -123,14 +125,13 @@ class Builds extends Action * @param Document $deployment * @param Document $template * @param Log $log + * @param Executor $executor * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log, Executor $executor): void { - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); - $functionId = $function->getId(); $log->addTag('functionId', $function->getId()); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 09c60c7740..5092599d67 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -55,12 +55,13 @@ class Deletes extends Action ->inject('deviceForBuilds') ->inject('deviceForCache') ->inject('certificates') + ->inject('executor') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executor, $executionRetention, $auditRetention, $log) ); } @@ -68,7 +69,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -93,10 +94,10 @@ class Deletes extends Action $this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); + $this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project, $executor); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); + $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project, $executor); break; case DELETE_TYPE_USERS: $this->deleteUser($getProjectDB, $document, $project); @@ -827,10 +828,11 @@ class Deletes extends Action * @param Device $deviceForBuilds * @param Document $document function document * @param Document $project + * @param Executor $executor * @return void * @throws Exception */ - private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void + private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project, Executor $executor): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -922,7 +924,7 @@ class Deletes extends Action * Request executor to delete all deployment containers */ Console::info("Requesting executor to delete all deployment containers for function " . $functionId); - $this->deleteRuntimes($getProjectDB, $document, $project); + $this->deleteRuntimes($getProjectDB, $document, $project, $executor); } /** @@ -993,10 +995,11 @@ class Deletes extends Action * @param Device $deviceForBuilds * @param Document $document * @param Document $project + * @param Executor $executor * @return void * @throws Exception */ - private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void + private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project, Executor $executor): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -1024,7 +1027,7 @@ class Deletes extends Action * Request executor to delete all deployment containers */ Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId); - $this->deleteRuntimes($getProjectDB, $document, $project); + $this->deleteRuntimes($getProjectDB, $document, $project, $executor); } /** @@ -1180,13 +1183,12 @@ class Deletes extends Action * @param callable $getProjectDB * @param ?Document $function * @param Document $project + * @param Executor $executor * @return void * @throws Exception */ - private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project): void + private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project, Executor $executor): void { - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); - $deleteByFunction = function (Document $function) use ($getProjectDB, $project, $executor) { $this->listByGroup( 'deployments', diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index a7caa3207f..4e1794c085 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -51,11 +51,12 @@ class Functions extends Action ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('log') + ->inject('executor') ->inject('isResourceBlocked') - ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); + ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $executor, $isResourceBlocked)); } - public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void + public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked): void { $payload = $message->getPayload() ?? []; @@ -146,6 +147,7 @@ class Functions extends Action queueForEvents: $queueForEvents, project: $project, function: $function, + executor: $executor, trigger: 'event', path: '/', method: 'POST', @@ -188,6 +190,7 @@ class Functions extends Action queueForEvents: $queueForEvents, project: $project, function: $function, + executor: $executor, trigger: 'http', path: $path, method: $method, @@ -212,6 +215,7 @@ class Functions extends Action queueForEvents: $queueForEvents, project: $project, function: $function, + executor: $executor, trigger: 'schedule', path: $path, method: $method, @@ -298,6 +302,7 @@ class Functions extends Action * @param Event $queueForEvents * @param Document $project * @param Document $function + * @param Executor $executor * @param string $trigger * @param string $path * @param string $method @@ -324,6 +329,7 @@ class Functions extends Action Event $queueForEvents, Document $project, Document $function, + Executor $executor, string $trigger, string $path, string $method, @@ -514,7 +520,6 @@ class Functions extends Action try { $version = $function->getAttribute('version', 'v2'); $command = $runtime['startCommand']; - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); $command = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $command . '"'; $executionResponse = $executor->createExecution( projectId: $project->getId(), diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 936fba5b24..6bc2fe7aab 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -21,17 +21,19 @@ class Executor private bool $selfSigned = false; - private string $endpoint; + /** + * @var callable(string, string): string $endpoint + */ + private $endpointSelector; protected array $headers; - public function __construct(string $endpoint) + /** + * @param callable(string, string): string $endpointSelector + */ + public function __construct(callable $endpointSelector) { - if (!filter_var($endpoint, FILTER_VALIDATE_URL)) { - throw new Exception('Unsupported endpoint'); - } - - $this->endpoint = $endpoint; + $this->endpointSelector = $endpointSelector; $this->headers = [ 'content-type' => 'application/json', 'authorization' => 'Bearer ' . System::getEnv('_APP_EXECUTOR_SECRET', ''), @@ -92,7 +94,8 @@ class Executor 'timeout' => $timeout, ]; - $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout); + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout); $status = $response['headers']['status-code']; if ($status >= 400) { @@ -123,7 +126,8 @@ class Executor 'timeout' => $timeout ]; - $this->call(self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback); + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $this->call($endpoint, self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback); } /** @@ -139,7 +143,8 @@ class Executor $runtimeId = "$projectId-$deploymentId"; $route = "/runtimes/$runtimeId"; - $response = $this->call(self::METHOD_DELETE, $route, [ + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $response = $this->call($endpoint, self::METHOD_DELETE, $route, [ 'x-opr-addressing-method' => 'broadcast' ], [], true, 30); @@ -227,7 +232,8 @@ class Executor $requestTimeout = $timeout + 15; } - $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout); + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout); $status = $response['headers']['status-code']; if ($status >= 400) { @@ -260,10 +266,10 @@ class Executor * @return array|string * @throws Exception */ - public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null) + private function call(string $endpoint, string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null) { $headers = array_merge($this->headers, $headers); - $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); + $ch = curl_init($endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); $responseHeaders = []; $responseStatus = -1; $responseType = ''; @@ -426,4 +432,9 @@ class Executor return $output; } + + private function selectEndpoint(string $projectId, string $deploymentId): string + { + return call_user_func($this->endpointSelector, $projectId, $deploymentId); + } }