From 831122679fb2d91ac6a08f920f484d265ba9a939 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 26 Oct 2023 17:45:23 +0300 Subject: [PATCH 1/8] creating script --- Dockerfile | 1 + app/console | 2 +- bin/delete-orphaned-projects | 3 + src/Appwrite/Platform/Services/Tasks.php | 3 + .../Platform/Tasks/DeleteOrphanedProjects.php | 108 ++++++++++++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 bin/delete-orphaned-projects create mode 100644 src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php diff --git a/Dockerfile b/Dockerfile index 8cea481543..33b1434659 100755 --- a/Dockerfile +++ b/Dockerfile @@ -101,6 +101,7 @@ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ chmod +x /usr/local/bin/patch-delete-project-collections && \ + chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ chmod +x /usr/local/bin/calc-tier-stats diff --git a/app/console b/app/console index fe835e5032..e965738987 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit fe835e50328ed80f67c66d2d449c0f7b51ade544 +Subproject commit e9657389879c8d76a9b3a0d3486c1d86f43c3bb9 diff --git a/bin/delete-orphaned-projects b/bin/delete-orphaned-projects new file mode 100644 index 0000000000..16b9e3184d --- /dev/null +++ b/bin/delete-orphaned-projects @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php delete-orphaned-projects $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 00779084d4..e725ff5f3e 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -18,6 +18,7 @@ use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; +use Appwrite\Platform\Tasks\DeleteOrphanedProjects; class Tasks extends Service { @@ -40,6 +41,8 @@ class Tasks extends Service ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) + ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) + ; } } diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php new file mode 100644 index 0000000000..88f4ae8ce2 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -0,0 +1,108 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + //docker compose exec -t appwrite delete-orphaned-projects + + Console::title('Delete orphaned projects V1'); + Console::success(APP_NAME . ' Delete orphaned projects started'); + + /** @var array $collections */ + $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + $projects = [$console]; + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + $collectionsCreated = $dbForProject->count(Database::METADATA); + } catch (\Throwable $th) { + $dbForConsole->deleteDocument('projects', $project->getId()); + Console::success('Deleting project (' . $project->getId() . ')'); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + } +} From 0f43aa68723518d25bb826ab04b81f72ed263121 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 26 Oct 2023 20:53:36 +0300 Subject: [PATCH 2/8] dry run --- .../Platform/Tasks/DeleteOrphanedProjects.php | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 88f4ae8ce2..91c8d6cc9c 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -55,6 +55,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); + $orphans = 0; $count = 0; $limit = 30; $sum = 30; @@ -69,8 +70,6 @@ class DeleteOrphanedProjects extends Action continue; } - Console::info("Getting stats for {$project->getId()}"); - try { $db = $project->getAttribute('database'); $adapter = $pools @@ -82,9 +81,18 @@ class DeleteOrphanedProjects extends Action $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = $dbForProject->count(Database::METADATA); + $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; + if ($collectionsCreated < ($totalProjects + 2)) { + Console::error($message); + $orphans++; + } else { + Console::log($message); + } } catch (\Throwable $th) { - $dbForConsole->deleteDocument('projects', $project->getId()); - Console::success('Deleting project (' . $project->getId() . ')'); + //$dbForConsole->deleteDocument('projects', $project->getId()); + //Console::success('Deleting project (' . $project->getId() . ')'); + Console::error(' (0) collections where found for project (' . $project->getId() . ')'); + $orphans++; } finally { $pools ->get($db) @@ -103,6 +111,6 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); } } From 7e6378b23b0d36b8ab9cc25b36a3ba55c3fc6d70 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 26 Oct 2023 20:56:32 +0300 Subject: [PATCH 3/8] dry run --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 91c8d6cc9c..4964276cf8 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -91,6 +91,7 @@ class DeleteOrphanedProjects extends Action } catch (\Throwable $th) { //$dbForConsole->deleteDocument('projects', $project->getId()); //Console::success('Deleting project (' . $project->getId() . ')'); + Console::error(' (0) collections where found for project (' . $project->getId() . ')'); $orphans++; } finally { From fc348b2055ca8dedde16a6649d5835e7f7e61e16 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 26 Oct 2023 21:12:13 +0300 Subject: [PATCH 4/8] dry run --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 4964276cf8..91c8d6cc9c 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -91,7 +91,6 @@ class DeleteOrphanedProjects extends Action } catch (\Throwable $th) { //$dbForConsole->deleteDocument('projects', $project->getId()); //Console::success('Deleting project (' . $project->getId() . ')'); - Console::error(' (0) collections where found for project (' . $project->getId() . ')'); $orphans++; } finally { From 45a0954b2465779115666fb5afa6d3418a3f2cf4 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 26 Oct 2023 21:12:43 +0300 Subject: [PATCH 5/8] dry run --- app/console | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/console b/app/console index e965738987..fe835e5032 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit e9657389879c8d76a9b3a0d3486c1d86f43c3bb9 +Subproject commit fe835e50328ed80f67c66d2d449c0f7b51ade544 From 07ef5535efaeb069fd94ee06e32005a4b91f7966 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 27 Oct 2023 18:23:41 +0300 Subject: [PATCH 6/8] dry run --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 91c8d6cc9c..26898e4797 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -38,7 +38,6 @@ class DeleteOrphanedProjects extends Action public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void { - //docker compose exec -t appwrite delete-orphaned-projects Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); From 917a4e9ab145d3156b3706a14357d3707927d223 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 27 Oct 2023 19:06:16 +0300 Subject: [PATCH 7/8] $collectionsConfig fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 26898e4797..159091d673 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -81,7 +81,7 @@ class DeleteOrphanedProjects extends Action $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = $dbForProject->count(Database::METADATA); $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; - if ($collectionsCreated < ($totalProjects + 2)) { + if ($collectionsCreated < ($collectionsConfig + 2)) { Console::error($message); $orphans++; } else { From 75484fa018be2fac8b25e685bb22f56289b46528 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 27 Oct 2023 19:07:27 +0300 Subject: [PATCH 8/8] $collectionsConfig fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 159091d673..2824f4e286 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -81,7 +81,7 @@ class DeleteOrphanedProjects extends Action $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = $dbForProject->count(Database::METADATA); $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; - if ($collectionsCreated < ($collectionsConfig + 2)) { + if ($collectionsCreated < (count($collectionsConfig) + 2)) { Console::error($message); $orphans++; } else {