From 56a8e38890e689772402095785f5a1f91eefa3a2 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 6 May 2025 10:50:56 +0300 Subject: [PATCH 1/6] listCollections with big limit --- src/Appwrite/Platform/Workers/Deletes.php | 48 +++++++++-------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index a61db63de6..9e1543b8b6 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -497,45 +497,33 @@ class Deletes extends Action AbuseDatabase::COLLECTION, ]; - $limit = \count($projectCollectionIds) + 25; - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); $projectTables = !\in_array($dsn->getHost(), $sharedTables); $sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1); $sharedTablesV2 = !$projectTables && !$sharedTablesV1; - $sharedTables = $sharedTablesV1 || $sharedTablesV2; - while (true) { - $collections = $dbForProject->listCollections($limit); + /** + * Consider using cursor + */ + $collections = $dbForProject->listCollections(PHP_INT_MAX); - foreach ($collections as $collection) { - try { - if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { - $dbForProject->deleteCollection($collection->getId()); - } else { - $this->deleteByGroup( - $collection->getId(), - [ - Query::orderAsc() - ], - database: $dbForProject - ); - } - } catch (Throwable $e) { - Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); + foreach ($collections as $collection) { + try { + if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { + $dbForProject->deleteCollection($collection->getId()); + } else { + $this->deleteByGroup( + $collection->getId(), + [ + Query::orderAsc() + ], + database: $dbForProject + ); } - } - - if ($sharedTables) { - $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections); - - if (empty(\array_diff($collectionsIds, $projectCollectionIds))) { - break; - } - } elseif (empty($collections)) { - break; + } catch (Throwable $e) { + Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); } } From 6264e616d3598e8e5fee1cdf5f0ba9b739f306f6 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 6 May 2025 12:18:29 +0300 Subject: [PATCH 2/6] Delete using Cursor --- src/Appwrite/Platform/Workers/Deletes.php | 29 +++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9e1543b8b6..50e09927c8 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -505,11 +505,11 @@ class Deletes extends Action $sharedTablesV2 = !$projectTables && !$sharedTablesV1; /** - * Consider using cursor + * @var $dbForProject Database */ - $collections = $dbForProject->listCollections(PHP_INT_MAX); - - foreach ($collections as $collection) { + var_dump($projectTables); + var_dump($projectCollectionIds); + $dbForProject->foreach(Database::METADATA, function (Document $collection) use ($dbForProject, $projectTables, $projectCollectionIds) { try { if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); @@ -525,7 +525,26 @@ class Deletes extends Action } catch (Throwable $e) { Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); } - } + }); + + // $collections = $dbForProject->listCollections(PHP_INT_MAX); +// foreach ($collections as $collection) { +// try { +// if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { +// $dbForProject->deleteCollection($collection->getId()); +// } else { +// $this->deleteByGroup( +// $collection->getId(), +// [ +// Query::orderAsc() +// ], +// database: $dbForProject +// ); +// } +// } catch (Throwable $e) { +// Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); +// } +// } // Delete Platforms $this->deleteByGroup('platforms', [ From abc2795daa24b70a1f3a5368177f7259799cc325 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 6 May 2025 12:25:56 +0300 Subject: [PATCH 3/6] Delete using Cursor --- src/Appwrite/Platform/Workers/Deletes.php | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 50e09927c8..6d796f3dda 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -507,8 +507,6 @@ class Deletes extends Action /** * @var $dbForProject Database */ - var_dump($projectTables); - var_dump($projectCollectionIds); $dbForProject->foreach(Database::METADATA, function (Document $collection) use ($dbForProject, $projectTables, $projectCollectionIds) { try { if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { @@ -527,25 +525,6 @@ class Deletes extends Action } }); - // $collections = $dbForProject->listCollections(PHP_INT_MAX); -// foreach ($collections as $collection) { -// try { -// if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { -// $dbForProject->deleteCollection($collection->getId()); -// } else { -// $this->deleteByGroup( -// $collection->getId(), -// [ -// Query::orderAsc() -// ], -// database: $dbForProject -// ); -// } -// } catch (Throwable $e) { -// Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); -// } -// } - // Delete Platforms $this->deleteByGroup('platforms', [ Query::equal('projectInternalId', [$projectInternalId]), From 6f34592efe055923c1b67ee6efb6ccd8ec936f6f Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 6 May 2025 14:18:26 +0300 Subject: [PATCH 4/6] Add messages --- src/Appwrite/Platform/Workers/Deletes.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6d796f3dda..3ebad0363a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -510,8 +510,10 @@ class Deletes extends Action $dbForProject->foreach(Database::METADATA, function (Document $collection) use ($dbForProject, $projectTables, $projectCollectionIds) { try { if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { + Console::info('DeleteProject deleteCollection "'.$collection->getId().'"'); $dbForProject->deleteCollection($collection->getId()); } else { + Console::info('DeleteProject deleteByGroup "'.$collection->getId().'"'); $this->deleteByGroup( $collection->getId(), [ From c5086d8fba8b90e0e7c4d0ffaf9c2843091cdbde Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 6 May 2025 15:18:51 +0300 Subject: [PATCH 5/6] Different color --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 3ebad0363a..7003f2b135 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -513,7 +513,7 @@ class Deletes extends Action Console::info('DeleteProject deleteCollection "'.$collection->getId().'"'); $dbForProject->deleteCollection($collection->getId()); } else { - Console::info('DeleteProject deleteByGroup "'.$collection->getId().'"'); + Console::log('DeleteProject deleteByGroup "'.$collection->getId().'"'); $this->deleteByGroup( $collection->getId(), [ From e5a1af3183eeeb3fb0586599fa6a12ca98e4453f Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 8 May 2025 10:43:28 +0300 Subject: [PATCH 6/6] Remove DBG --- src/Appwrite/Platform/Workers/Deletes.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 7003f2b135..6d796f3dda 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -510,10 +510,8 @@ class Deletes extends Action $dbForProject->foreach(Database::METADATA, function (Document $collection) use ($dbForProject, $projectTables, $projectCollectionIds) { try { if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { - Console::info('DeleteProject deleteCollection "'.$collection->getId().'"'); $dbForProject->deleteCollection($collection->getId()); } else { - Console::log('DeleteProject deleteByGroup "'.$collection->getId().'"'); $this->deleteByGroup( $collection->getId(), [