Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-24 01:18:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Index check for delete worker

Browse source
This commit is contained in:
fogelito 2025-03-17 17:05:40 +02:00
parent 5a5d43f3d0
commit bbfd22ce2e
1 changed files with 113 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

147
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -248,7 +248,8 @@ class Deletes extends Action
$this->deleteByGroup( $this->deleteByGroup(
'subscribers', 'subscribers',
[ [
Query::equal('topicInternalId', [$topic->getInternalId()]) Query::equal('topicInternalId', [$topic->getInternalId()]),
Query::orderAsc(),
], ],
$getProjectDB($project) $getProjectDB($project)
); );
@ -269,7 +270,8 @@ class Deletes extends Action
$this->deleteByGroup( $this->deleteByGroup(
'subscribers', 'subscribers',
[ [
Query::equal('targetInternalId', [$target->getInternalId()]) Query::equal('targetInternalId', [$target->getInternalId()]),
Query::orderAsc(),
], ],
$dbForProject, $dbForProject,
function (Document $subscriber) use ($dbForProject, $target) { function (Document $subscriber) use ($dbForProject, $target) {
@ -303,10 +305,14 @@ class Deletes extends Action
*/ */
private function deleteExpiredTargets(Document $project, callable $getProjectDB): void private function deleteExpiredTargets(Document $project, callable $getProjectDB): void
{ {
/**
* todo: No index found for `expired` attribute
*/
$this->deleteByGroup( $this->deleteByGroup(
'targets', 'targets',
[ [
Query::equal('expired', [true]) Query::equal('expired', [true]),
Query::orderAsc(),
], ],
$getProjectDB($project), $getProjectDB($project),
function (Document $target) use ($getProjectDB, $project) { function (Document $target) use ($getProjectDB, $project) {
@ -317,10 +323,14 @@ class Deletes extends Action
private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void
{ {
/**
* todo: No index found for `sessionInternalId` attribute
*/
$this->deleteByGroup( $this->deleteByGroup(
'targets', 'targets',
[ [
Query::equal('sessionInternalId', [$session->getInternalId()]) Query::equal('sessionInternalId', [$session->getInternalId()]),
Query::orderAsc(),
], ],
$getProjectDB($project), $getProjectDB($project),
function (Document $target) use ($getProjectDB, $project) { function (Document $target) use ($getProjectDB, $project) {
@ -348,10 +358,17 @@ class Deletes extends Action
); );
$query[] = Query::equal('resource', [$resource]); $query[] = Query::equal('resource', [$resource]);
if (!empty($resourceType)) { if (!empty($resourceType)) {
$query[] = Query::equal('resourceType', [$resourceType]); $query[] = Query::equal('resourceType', [$resourceType]);
} }
$query[] = Query::orderAsc();
/**
* todo: missing index on `resource`, `resourceType`
*/
$this->deleteByGroup( $this->deleteByGroup(
'cache', 'cache',
$query, $query,
@ -388,7 +405,7 @@ class Deletes extends Action
$query = [ $query = [
Query::lessThan('accessedAt', $datetime), Query::lessThan('accessedAt', $datetime),
Query::orderDesc('accessedAt'), Query::orderDesc('accessedAt'),
Query::orderDesc('$internalId'), Query::orderDesc(),
]; ];
$this->deleteByGroup( $this->deleteByGroup(
@ -421,10 +438,10 @@ class Deletes extends Action
// Delete Usage stats from projectDB // Delete Usage stats from projectDB
$this->deleteByGroup('stats', [ $this->deleteByGroup('stats', [
Query::equal('period', ['1h']),
Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::orderDesc('time'), Query::orderDesc('time'),
Query::orderDesc('$internalId'), Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC
Query::equal('period', ['1h']),
], $dbForProject); ], $dbForProject);
if ($project->getId() !== 'console') { if ($project->getId() !== 'console') {
@ -433,10 +450,10 @@ class Deletes extends Action
// Delete Usage stats from logsDB // Delete Usage stats from logsDB
$this->deleteByGroup('stats', [ $this->deleteByGroup('stats', [
Query::equal('period', ['1h']),
Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::orderDesc('time'), Query::orderDesc('time'),
Query::orderDesc('$internalId'), Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC
Query::equal('period', ['1h']),
], $dbForLogs); ], $dbForLogs);
} }
} }
@ -457,7 +474,8 @@ class Deletes extends Action
$this->deleteByGroup( $this->deleteByGroup(
'memberships', 'memberships',
[ [
Query::equal('teamInternalId', [$teamInternalId]) Query::equal('teamInternalId', [$teamInternalId]),
Query::orderAsc()
], ],
$dbForProject, $dbForProject,
function (Document $membership) use ($dbForProject) { function (Document $membership) use ($dbForProject) {
@ -547,7 +565,13 @@ class Deletes extends Action
if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) {
$dbForProject->deleteCollection($collection->getId()); $dbForProject->deleteCollection($collection->getId());
} else { } else {
$this->deleteByGroup($collection->getId(), [], database: $dbForProject); $this->deleteByGroup(
$collection->getId(),
[
Query::orderAsc()
],
database: $dbForProject
);
} }
} catch (Throwable $e) { } catch (Throwable $e) {
Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
@ -567,58 +591,82 @@ class Deletes extends Action
// Delete Platforms // Delete Platforms
$this->deleteByGroup('platforms', [ $this->deleteByGroup('platforms', [
Query::equal('projectInternalId', [$projectInternalId]) Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
// Delete project and function rules // Delete project and function rules
$this->deleteByGroup('rules', [ $this->deleteByGroup('rules', [
Query::equal('projectInternalId', [$projectInternalId]) Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) { ], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) {
$this->deleteRule($dbForPlatform, $document, $certificates); $this->deleteRule($dbForPlatform, $document, $certificates);
}); });
// Delete Keys // Delete Keys
$this->deleteByGroup('keys', [ $this->deleteByGroup('keys', [
Query::equal('projectInternalId', [$projectInternalId]) Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
// Delete Webhooks // Delete Webhooks
$this->deleteByGroup('webhooks', [ $this->deleteByGroup('webhooks', [
Query::equal('projectInternalId', [$projectInternalId]) Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
// Delete VCS Installations // Delete VCS Installations
$this->deleteByGroup('installations', [ $this->deleteByGroup('installations', [
Query::equal('projectInternalId', [$projectInternalId]) Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
// Delete VCS Repositories // Delete VCS Repositories
$this->deleteByGroup('repositories', [ $this->deleteByGroup('repositories', [
Query::equal('projectInternalId', [$projectInternalId]), Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
// Delete VCS comments // Delete VCS comments
$this->deleteByGroup('vcsComments', [ $this->deleteByGroup('vcsComments', [
Query::equal('projectInternalId', [$projectInternalId]), Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
// Delete Schedules (No projectInternalId in this collection) /**
* No projectInternalId in this collection
* todo: No index for `projectId` attribute
*/
// Delete Schedules ()
$this->deleteByGroup('schedules', [ $this->deleteByGroup('schedules', [
Query::equal('projectId', [$projectId]), Query::equal('projectId', [$projectId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
// Delete metadata table // Delete metadata table
if ($projectTables) { if ($projectTables) {
$dbForProject->deleteCollection(Database::METADATA); $dbForProject->deleteCollection(Database::METADATA);
} elseif ($sharedTablesV1) { } elseif ($sharedTablesV1) {
$this->deleteByGroup(Database::METADATA, [], $dbForProject); $this->deleteByGroup(
Database::METADATA,
[
Query::orderAsc()
],
$dbForProject
);
} elseif ($sharedTablesV2) { } elseif ($sharedTablesV2) {
$queries = \array_map( $queries = \array_map(
fn ($id) => Query::notEqual('$id', $id), fn ($id) => Query::notEqual('$id', $id),
$projectCollectionIds $projectCollectionIds
); );
$this->deleteByGroup(Database::METADATA, $queries, $dbForProject); $queries[] = Query::orderAsc();
$this->deleteByGroup(
Database::METADATA,
$queries,
$dbForProject
);
} }
// Delete all storage directories // Delete all storage directories
@ -643,14 +691,16 @@ class Deletes extends Action
// Delete all sessions of this user from the sessions table and update the sessions field of the user record // Delete all sessions of this user from the sessions table and update the sessions field of the user record
$this->deleteByGroup('sessions', [ $this->deleteByGroup('sessions', [
Query::equal('userInternalId', [$userInternalId]) Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject); ], $dbForProject);
$dbForProject->purgeCachedDocument('users', $userId); $dbForProject->purgeCachedDocument('users', $userId);
// Delete Memberships and decrement team membership counts // Delete Memberships and decrement team membership counts
$this->deleteByGroup('memberships', [ $this->deleteByGroup('memberships', [
Query::equal('userInternalId', [$userInternalId]) Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($dbForProject) { ], $dbForProject, function (Document $document) use ($dbForProject) {
if ($document->getAttribute('confirm')) { // Count only confirmed members if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId'); $teamId = $document->getAttribute('teamId');
@ -663,19 +713,25 @@ class Deletes extends Action
// Delete tokens // Delete tokens
$this->deleteByGroup('tokens', [ $this->deleteByGroup('tokens', [
Query::equal('userInternalId', [$userInternalId]) Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject); ], $dbForProject);
// Delete identities // Delete identities
/**
* todo: Remove Duplication index `_key_userInternalId`
*/
$this->deleteByGroup('identities', [ $this->deleteByGroup('identities', [
Query::equal('userInternalId', [$userInternalId]) Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject); ], $dbForProject);
// Delete targets // Delete targets
$this->deleteByGroup( $this->deleteByGroup(
'targets', 'targets',
[ [
Query::equal('userInternalId', [$userInternalId]) Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], ],
$dbForProject, $dbForProject,
function (Document $target) use ($getProjectDB, $project) { function (Document $target) use ($getProjectDB, $project) {
@ -699,7 +755,7 @@ class Deletes extends Action
$this->deleteByGroup('executions', [ $this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime), Query::lessThan('$createdAt', $datetime),
Query::orderDesc('$createdAt'), Query::orderDesc('$createdAt'),
Query::orderDesc('$internalId'), Query::orderDesc(),
], $dbForProject); ], $dbForProject);
} }
@ -719,7 +775,7 @@ class Deletes extends Action
$this->deleteByGroup('sessions', [ $this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired), Query::lessThan('$createdAt', $expired),
Query::orderDesc('$createdAt'), Query::orderDesc('$createdAt'),
Query::orderDesc('$internalId'), Query::orderDesc(),
], $dbForProject); ], $dbForProject);
} }
@ -735,7 +791,7 @@ class Deletes extends Action
$this->deleteByGroup('realtime', [ $this->deleteByGroup('realtime', [
Query::lessThan('timestamp', $datetime), Query::lessThan('timestamp', $datetime),
Query::orderDesc('timestamp'), Query::orderDesc('timestamp'),
Query::orderDesc('$internalId'), Query::orderAsc(), // KEY "_key_timestamp" ("timestamp" DESC),
], $dbForPlatform); ], $dbForPlatform);
} }
@ -755,7 +811,7 @@ class Deletes extends Action
$this->deleteByGroup(Audit::COLLECTION, [ $this->deleteByGroup(Audit::COLLECTION, [
Query::lessThan('time', $auditRetention), Query::lessThan('time', $auditRetention),
Query::orderDesc('time'), Query::orderDesc('time'),
Query::orderDesc('$internalId'), Query::orderAsc(), // KEY "index-time" ("time" DESC)
], $dbForProject); ], $dbForProject);
} catch (DatabaseException $e) { } catch (DatabaseException $e) {
Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage());
@ -780,33 +836,39 @@ class Deletes extends Action
/** /**
* Delete rules * Delete rules
* todo: No index for this query
*/ */
Console::info("Deleting rules for function " . $functionId); Console::info("Deleting rules for function " . $functionId);
$this->deleteByGroup('rules', [ $this->deleteByGroup('rules', [
Query::equal('resourceType', ['function']), Query::equal('resourceType', ['function']),
Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('projectInternalId', [$project->getInternalId()]) Query::equal('projectInternalId', [$project->getInternalId()]),
Query::orderAsc()
], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) {
$this->deleteRule($dbForPlatform, $document, $certificates); $this->deleteRule($dbForPlatform, $document, $certificates);
}); });
/** /**
* Delete Variables * Delete Variables
* todo: No index for this query
*/ */
Console::info("Deleting variables for function " . $functionId); Console::info("Deleting variables for function " . $functionId);
$this->deleteByGroup('variables', [ $this->deleteByGroup('variables', [
Query::equal('resourceType', ['function']), Query::equal('resourceType', ['function']),
Query::equal('resourceInternalId', [$functionInternalId]) Query::equal('resourceInternalId', [$functionInternalId]),
Query::orderAsc()
], $dbForProject); ], $dbForProject);
/** /**
* Delete Deployments * Delete Deployments
* todo: No index for `resourceInternalId`, perhaps use resourceId until fixed
*/ */
Console::info("Deleting deployments for function " . $functionId); Console::info("Deleting deployments for function " . $functionId);
$deploymentInternalIds = []; $deploymentInternalIds = [];
$this->deleteByGroup('deployments', [ $this->deleteByGroup('deployments', [
Query::equal('resourceInternalId', [$functionInternalId]) Query::equal('resourceInternalId', [$functionInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) { ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) {
$deploymentInternalIds[] = $document->getInternalId(); $deploymentInternalIds[] = $document->getInternalId();
$this->deleteDeploymentFiles($deviceForFunctions, $document); $this->deleteDeploymentFiles($deviceForFunctions, $document);
@ -814,12 +876,14 @@ class Deletes extends Action
/** /**
* Delete builds * Delete builds
* todo: No index for `deploymentInternalId`, perhaps use deploymentId until fixed
*/ */
Console::info("Deleting builds for function " . $functionId); Console::info("Deleting builds for function " . $functionId);
foreach ($deploymentInternalIds as $deploymentInternalId) { foreach ($deploymentInternalIds as $deploymentInternalId) {
$this->deleteByGroup('builds', [ $this->deleteByGroup('builds', [
Query::equal('deploymentInternalId', [$deploymentInternalId]) Query::equal('deploymentInternalId', [$deploymentInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($deviceForBuilds) { ], $dbForProject, function (Document $document) use ($deviceForBuilds) {
$this->deleteBuildFiles($deviceForBuilds, $document); $this->deleteBuildFiles($deviceForBuilds, $document);
}); });
@ -827,26 +891,35 @@ class Deletes extends Action
/** /**
* Delete Executions * Delete Executions
* todo: No index for `functionInternalId` , perhaps use functionId until fixed
*/ */
Console::info("Deleting executions for function " . $functionId); Console::info("Deleting executions for function " . $functionId);
$this->deleteByGroup('executions', [ $this->deleteByGroup('executions', [
Query::equal('functionInternalId', [$functionInternalId]) Query::equal('functionInternalId', [$functionInternalId]),
Query::orderAsc()
], $dbForProject); ], $dbForProject);
/** /**
* Delete VCS Repositories and VCS Comments * Delete VCS Repositories and VCS Comments
* todo: no index for this query
*/ */
Console::info("Deleting VCS repositories and comments linked to function " . $functionId); Console::info("Deleting VCS repositories and comments linked to function " . $functionId);
$this->deleteByGroup('repositories', [ $this->deleteByGroup('repositories', [
Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('projectInternalId', [$project->getInternalId()]),
Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('resourceType', ['function']), Query::equal('resourceType', ['function']),
Query::orderAsc()
], $dbForPlatform, function (Document $document) use ($dbForPlatform) { ], $dbForPlatform, function (Document $document) use ($dbForPlatform) {
$providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $providerRepositoryId = $document->getAttribute('providerRepositoryId', '');
$projectInternalId = $document->getAttribute('projectInternalId', ''); $projectInternalId = $document->getAttribute('projectInternalId', '');
/**
* todo: add index to this query
*/
$this->deleteByGroup('vcsComments', [ $this->deleteByGroup('vcsComments', [
Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::equal('projectInternalId', [$projectInternalId]), Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform); ], $dbForPlatform);
}); });
@ -942,11 +1015,13 @@ class Deletes extends Action
/** /**
* Delete builds * Delete builds
* todo: no index for `deploymentInternalId` change to temporary to deploymentId?
*/ */
Console::info("Deleting builds for deployment " . $deploymentId); Console::info("Deleting builds for deployment " . $deploymentId);
$this->deleteByGroup('builds', [ $this->deleteByGroup('builds', [
Query::equal('deploymentInternalId', [$deploymentInternalId]) Query::equal('deploymentInternalId', [$deploymentInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($deviceForBuilds) { ], $dbForProject, function (Document $document) use ($deviceForBuilds) {
$this->deleteBuildFiles($deviceForBuilds, $document); $this->deleteBuildFiles($deviceForBuilds, $document);
}); });
@ -974,6 +1049,10 @@ class Deletes extends Action
): void { ): void {
$start = \microtime(true); $start = \microtime(true);
/**
* deleteDocuments uses a cursor, we need to add a unique order by field or use default
*/
try { try {
$documents = $database->deleteDocuments($collection, $queries); $documents = $database->deleteDocuments($collection, $queries);
} catch (Throwable $th) { } catch (Throwable $th) {