From 0f3c43897621e892e1b1dceed6378cf385112803 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 8 Nov 2023 12:05:51 +0200 Subject: [PATCH 1/8] added inf metric --- .../Platform/Tasks/DeleteOrphanedProjects.php | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2824f4e286..aeaab4b248 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -13,6 +13,8 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; +use Utopia\Validator\Boolean; +use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -26,25 +28,23 @@ class DeleteOrphanedProjects extends Action $this ->desc('Get stats for projects') + ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); + ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($commit, $pools, $cache, $dbForConsole, $register); }); } - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void { Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); - /** @var array $collections */ - $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; - /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -54,7 +54,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 0; + $orphans = 1; $count = 0; $limit = 30; $sum = 30; @@ -80,17 +80,22 @@ class DeleteOrphanedProjects extends Action $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = $dbForProject->count(Database::METADATA); - $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; - if ($collectionsCreated < (count($collectionsConfig) + 2)) { - Console::error($message); + if ($collectionsCreated === 0) { + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; - } else { - Console::log($message); } } catch (\Throwable $th) { - //$dbForConsole->deleteDocument('projects', $project->getId()); - //Console::success('Deleting project (' . $project->getId() . ')'); - Console::error(' (0) collections where found for project (' . $project->getId() . ')'); + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; } finally { $pools @@ -110,6 +115,15 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); + } + + private function deleteProject(Database $dbForConsole, $projectId): void + { + try { + $dbForConsole->deleteDocument('projects', $projectId); + } catch (\Throwable $th) { + Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); + } } } From b9007a59564f38b54cf90ba0d88aa62060d62fff Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 15:41:10 +0200 Subject: [PATCH 2/8] wrapping create stats query with Authorization::skip --- src/Appwrite/Usage/Calculators/TimeSeries.php | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index e0a12b443f..9ead48e93f 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -8,6 +8,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use InfluxDB\Database as InfluxDatabase; use DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; class TimeSeries extends Calculator @@ -422,14 +423,16 @@ class TimeSeries extends Calculator */ private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { + $id = \md5("{$time}_{$period}_{$metric}"); $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); - try { - $document = $database->getDocument('stats', $id); - if ($document->isEmpty()) { - $database->createDocument('stats', new Document([ + Authorization::skip(function () use ($database, $id, $period, $time, $metric, $value, $type, $projectId) { + try { + $document = $database->getDocument('stats', $id); + if ($document->isEmpty()) { + $database->createDocument('stats', new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -437,21 +440,22 @@ class TimeSeries extends Calculator 'value' => $value, 'type' => $type, 'region' => $this->region, - ])); - } else { - $database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); + ])); + } else { + $database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } + }); $this->register->get('pools')->reclaim(); } From dd0bf66212450041ab7518a0f958c8fdbb9c8166 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:24:55 +0200 Subject: [PATCH 3/8] delete orphaned projects task --- .../Platform/Tasks/DeleteOrphanedProjects.php | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index aeaab4b248..2af9c54a6e 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -2,11 +2,9 @@ namespace Appwrite\Platform\Tasks; -use PHPMailer\PHPMailer\PHPMailer; use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -14,7 +12,6 @@ use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Validator\Boolean; -use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -27,7 +24,7 @@ class DeleteOrphanedProjects extends Action { $this - ->desc('Get stats for projects') + ->desc('Delete orphaned projects') ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') @@ -45,6 +42,9 @@ class DeleteOrphanedProjects extends Action Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); + /** @var array $collections */ + $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -55,6 +55,7 @@ class DeleteOrphanedProjects extends Action Console::success("Found a total of: {$totalProjects} projects"); $orphans = 1; + $cnt = 0; $count = 0; $limit = 30; $sum = 30; @@ -79,24 +80,43 @@ class DeleteOrphanedProjects extends Action $dbForProject = new Database($adapter, $cache); $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); - $collectionsCreated = $dbForProject->count(Database::METADATA); - if ($collectionsCreated === 0) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + $collectionsCreated = 0; + $cnt++; + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + $collectionsCreated = $dbForProject->count(Database::METADATA); + } + + $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + /** + * +2 == audit+abuse + */ + if ($collectionsCreated === (count($collectionsConfig) + 2)) { + Console::log($msg . ' ignoring....'); + continue; + } + + Console::log($msg); + + if ($collectionsCreated > 0) { + $collections = $dbForProject->find(Database::METADATA, []); + foreach ($collections as $collection) { + if ($commit) { + $dbForProject->deleteCollection($collection->getId()); + $dbForConsole->deleteCachedCollection($collection->getId()); + } + Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); } + } + if ($commit) { + $dbForConsole->deleteDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); + } + + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + $orphans++; - } } catch (\Throwable $th) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); - } - $orphans++; + Console::error('Error: ' . $th->getMessage()); } finally { $pools ->get($db) @@ -117,13 +137,4 @@ class DeleteOrphanedProjects extends Action Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); } - - private function deleteProject(Database $dbForConsole, $projectId): void - { - try { - $dbForConsole->deleteDocument('projects', $projectId); - } catch (\Throwable $th) { - Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); - } - } } From 60c0f4c97393d8d665da5ab7d7b697fafa69abbc Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:27:20 +0200 Subject: [PATCH 4/8] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2af9c54a6e..2da3c91384 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -86,9 +86,9 @@ class DeleteOrphanedProjects extends Action $collectionsCreated = $dbForProject->count(Database::METADATA); } - $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; /** - * +2 == audit+abuse + * +2 = audit+abuse */ if ($collectionsCreated === (count($collectionsConfig) + 2)) { Console::log($msg . ' ignoring....'); From 3d9ee8bc521367d8242de2e4035970d44a9050da Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:28:48 +0200 Subject: [PATCH 5/8] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2da3c91384..ee9931bde4 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -25,7 +25,7 @@ class DeleteOrphanedProjects extends Action $this ->desc('Delete orphaned projects') - ->param('commit', false, new boolean(true), 'Commit project deletion', true) + ->param('commit', false, new Boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') From c2e5849c49738b59001d61606dc0a642b9702e0b Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 14 Nov 2023 16:49:45 +0200 Subject: [PATCH 6/8] removing blank line --- src/Appwrite/Usage/Calculators/TimeSeries.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 9ead48e93f..38574b9b03 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -423,7 +423,6 @@ class TimeSeries extends Calculator */ private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { - $id = \md5("{$time}_{$period}_{$metric}"); $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); From 7efecbca8b0637d0a16c7d78ded922d43a44a5d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Wed, 15 Nov 2023 17:20:50 +0000 Subject: [PATCH 7/8] chore: fixed indentation --- src/Appwrite/Usage/Calculators/TimeSeries.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 38574b9b03..6dbf49c44c 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -432,13 +432,13 @@ class TimeSeries extends Calculator $document = $database->getDocument('stats', $id); if ($document->isEmpty()) { $database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => $type, - 'region' => $this->region, + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => $type, + 'region' => $this->region, ])); } else { $database->updateDocument( From 6b5e734dfaedfb6fb291d7dab6a7a883dcc5a555 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 15 Nov 2023 20:09:50 +0200 Subject: [PATCH 8/8] indentation small fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index ee9931bde4..5f0ecbe1db 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -112,9 +112,9 @@ class DeleteOrphanedProjects extends Action $dbForConsole->deleteCachedDocument('projects', $project->getId()); } - Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); - $orphans++; + $orphans++; } catch (\Throwable $th) { Console::error('Error: ' . $th->getMessage()); } finally {