diff --git a/app/config/collections/common.php b/app/config/collections/common.php index f68400e226..3f6245dfdd 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1038,7 +1038,7 @@ return [ '$id' => ID::custom('providerUid'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 2048, // Decrease to 128 as in index length? 'signed' => true, 'required' => false, 'default' => null, @@ -1107,14 +1107,14 @@ return [ '$id' => ID::custom('_key_userInternalId_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['userInternalId', 'provider', 'providerUid'], - 'lengths' => [11, 128, 128], + 'lengths' => [11, 128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [128, 128], + 'lengths' => [128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ diff --git a/composer.json b/composer.json index d7b8505b5c..b1b6aed539 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.61.*", + "utopia-php/database": "0.62.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index bb5cc81fcc..6dff127784 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -248,7 +248,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('topicInternalId', [$topic->getInternalId()]) + Query::equal('topicInternalId', [$topic->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project) ); @@ -269,7 +270,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('targetInternalId', [$target->getInternalId()]) + Query::equal('targetInternalId', [$target->getInternalId()]), + Query::orderAsc(), ], $dbForProject, function (Document $subscriber) use ($dbForProject, $target) { @@ -306,7 +308,8 @@ class Deletes extends Action $this->deleteByGroup( 'targets', [ - Query::equal('expired', [true]) + Query::equal('expired', [true]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -320,7 +323,8 @@ class Deletes extends Action $this->deleteByGroup( 'targets', [ - Query::equal('sessionInternalId', [$session->getInternalId()]) + Query::equal('sessionInternalId', [$session->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -347,14 +351,20 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query[] = Query::equal('resource', [$resource]); + $queries = [ + Query::equal('resource', [$resource]) + ]; + if (!empty($resourceType)) { - $query[] = Query::equal('resourceType', [$resourceType]); + $queries[] = Query::equal('resourceType', [$resourceType]); } + $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); + $queries[] = Query::orderAsc(); + $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -385,15 +395,16 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query = [ + $queries = [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ]; $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -421,10 +432,11 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), ], $dbForProject); if ($project->getId() !== 'console') { @@ -433,10 +445,11 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), ], $dbForLogs); } } @@ -457,7 +470,8 @@ class Deletes extends Action $this->deleteByGroup( 'memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) + Query::equal('teamInternalId', [$teamInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $membership) use ($dbForProject) { @@ -547,7 +561,13 @@ class Deletes extends Action if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); } else { - $this->deleteByGroup($collection->getId(), [], database: $dbForProject); + $this->deleteByGroup( + $collection->getId(), + [ + Query::orderAsc() + ], + database: $dbForProject + ); } } catch (Throwable $e) { Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); @@ -567,58 +587,78 @@ class Deletes extends Action // Delete Platforms $this->deleteByGroup('platforms', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete project and function rules $this->deleteByGroup('rules', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); // Delete Keys $this->deleteByGroup('keys', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete Webhooks $this->deleteByGroup('webhooks', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Installations $this->deleteByGroup('installations', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Repositories $this->deleteByGroup('repositories', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS comments $this->deleteByGroup('vcsComments', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); - // Delete Schedules (No projectInternalId in this collection) + // Delete Schedules $this->deleteByGroup('schedules', [ Query::equal('projectId', [$projectId]), + Query::orderAsc() ], $dbForPlatform); // Delete metadata table if ($projectTables) { $dbForProject->deleteCollection(Database::METADATA); } elseif ($sharedTablesV1) { - $this->deleteByGroup(Database::METADATA, [], $dbForProject); + $this->deleteByGroup( + Database::METADATA, + [ + Query::orderAsc() + ], + $dbForProject + ); } elseif ($sharedTablesV2) { $queries = \array_map( fn ($id) => Query::notEqual('$id', $id), $projectCollectionIds ); - $this->deleteByGroup(Database::METADATA, $queries, $dbForProject); + $queries[] = Query::orderAsc(); + + $this->deleteByGroup( + Database::METADATA, + $queries, + $dbForProject + ); } // Delete all storage directories @@ -643,14 +683,16 @@ class Deletes extends Action // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); $dbForProject->purgeCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($dbForProject) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); @@ -663,19 +705,22 @@ class Deletes extends Action // Delete tokens $this->deleteByGroup('tokens', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete identities $this->deleteByGroup('identities', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete targets $this->deleteByGroup( 'targets', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $target) use ($getProjectDB, $project) { @@ -697,9 +742,10 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -717,9 +763,10 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -735,7 +782,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderDesc('$internalId'), + Query::orderAsc(), ], $dbForPlatform); } @@ -753,9 +800,10 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderDesc('$internalId'), + Query::orderAsc(), ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); @@ -783,9 +831,10 @@ class Deletes extends Action */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ - Query::equal('resourceType', ['function']), + Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('resourceInternalId', [$functionInternalId]), - Query::equal('projectInternalId', [$project->getInternalId()]) + Query::equal('resourceType', ['function']), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); @@ -795,8 +844,9 @@ class Deletes extends Action */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ + Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), - Query::equal('resourceInternalId', [$functionInternalId]) + Query::orderAsc() ], $dbForProject); /** @@ -806,7 +856,8 @@ class Deletes extends Action $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ - Query::equal('resourceInternalId', [$functionInternalId]) + Query::equal('resourceInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $this->deleteDeploymentFiles($deviceForFunctions, $document); @@ -819,7 +870,8 @@ class Deletes extends Action foreach ($deploymentInternalIds as $deploymentInternalId) { $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -830,7 +882,9 @@ class Deletes extends Action */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ - Query::equal('functionInternalId', [$functionInternalId]) + Query::select(['$internalId', '$id', '$updatedAt']), + Query::equal('functionInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject); /** @@ -841,12 +895,15 @@ class Deletes extends Action Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform) { $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $projectInternalId = $document->getAttribute('projectInternalId', ''); + $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); }); @@ -946,7 +1003,8 @@ class Deletes extends Action Console::info("Deleting builds for deployment " . $deploymentId); $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -974,6 +1032,10 @@ class Deletes extends Action ): void { $start = \microtime(true); + /** + * deleteDocuments uses a cursor, we need to add a unique order by field or use default + */ + try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index b8cff64214..e8eafba5a0 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -71,12 +71,18 @@ class Base extends Queries 'array' => false, ]); + $internalId = new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); + $validators = [ new Limit(), new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order($attributes), + new Order([...$attributes, $internalId]), ]; parent::__construct($validators);