From 9a12ab8f8a322fd91162ac8eda4c5c0a302c8c06 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 19:36:52 -0500 Subject: [PATCH 1/5] feat: add migration stats task --- Dockerfile | 3 +- bin/get-migration-stats | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/CalcTierStats.php | 2 +- .../Platform/Tasks/GetMigrationStats.php | 184 ++++++++++++++++++ 5 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 bin/get-migration-stats create mode 100644 src/Appwrite/Platform/Tasks/GetMigrationStats.php diff --git a/Dockerfile b/Dockerfile index 059c499bd9..8721301269 100755 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats + chmod +x /usr/local/bin/calc-tier-stats && \ + chmod +x /usr/local/bin/get-migration-stats # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/get-migration-stats b/bin/get-migration-stats new file mode 100644 index 0000000000..efc1934e15 --- /dev/null +++ b/bin/get-migration-stats @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php get-migration-stats $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 28d7046dd1..dc6ddc1a5b 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\DeleteOrphanedProjects; +use Appwrite\Platform\Tasks\GetMigrationStats; use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service @@ -44,6 +45,7 @@ class Tasks extends Service ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) + ->addAction(GetMigrationStats::getName(), new GetMigrationStats()) ; } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 2a2bc20af9..c579dd2627 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -200,7 +200,7 @@ class CalcTierStats extends Action try { /** Get Domains */ - $stats['Domains'] = $dbForConsole->count('domains', [ + $stats['Domains'] = $dbForConsole->count('rules', [ Query::equal('projectInternalId', [$project->getInternalId()]), ]); } catch (\Throwable) { diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php new file mode 100644 index 0000000000..bd7885c7ce --- /dev/null +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -0,0 +1,184 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + /** + * @throws \Utopia\Exception + * @throws CannotInsertRecord + */ + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + //docker compose exec -t appwrite get-migration-stats + + Console::title('Migration stats calculation V1'); + Console::success(APP_NAME . ' Migration stats calculation has started'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/migration_stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Migration details */ + $migrations = $dbForProject->find('migrations', [ + Query::limit(500) + ]); + + $migrations = array_map(function ($migration) use ($project) { + $result['Project ID'] = $project->getId(); + $result['$id'] = $migration->getAttribute('$id'); + $result['$createdAt'] = $migration->getAttribute('$createdAt'); + $result['status'] = $migration->getAttribute('status'); + $result['stage'] = $migration->getAttribute('stage'); + $result['source'] = $migration->getAttribute('source'); + return array_values($result); + }, $migrations); + + $csv->insertAll($migrations); + } catch (\Throwable $th) { + Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + + $pools + ->get('console') + ->reclaim(); + + /** @var PHPMailer $mail */ + $mail = $register->get('smtp'); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); + + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Migration Report for {$this->date}"; + $mail->Body = "Please find the migration report atttached"; + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } +} From 28c61acec68a4960c553e27611e87da0a7abc2b1 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 19:39:07 -0500 Subject: [PATCH 2/5] feat: update migration task --- src/Appwrite/Platform/Tasks/CalcTierStats.php | 4 ++-- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index c579dd2627..ef900aab34 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -99,8 +99,8 @@ class CalcTierStats extends Action $projects = [$console]; $count = 0; - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while (!empty($projects)) { foreach ($projects as $project) { diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index bd7885c7ce..3ef4f03d25 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -79,8 +79,8 @@ class GetMigrationStats extends Action $projects = [$console]; $count = 0; - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while (!empty($projects)) { foreach ($projects as $project) { @@ -123,7 +123,9 @@ class GetMigrationStats extends Action return array_values($result); }, $migrations); - $csv->insertAll($migrations); + if(!empty($migrations)) { + $csv->insertAll($migrations); + } } catch (\Throwable $th) { Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); } finally { From f81d6756097eea3fbf3c982dbdcdfca46b570d23 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 20:10:24 -0500 Subject: [PATCH 3/5] feat: run linter --- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index 3ef4f03d25..6fddf2d7e0 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -123,7 +123,7 @@ class GetMigrationStats extends Action return array_values($result); }, $migrations); - if(!empty($migrations)) { + if (!empty($migrations)) { $csv->insertAll($migrations); } } catch (\Throwable $th) { From 9d31d8d950c972a86c797f43ae2261dbd0512902 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 20:10:56 -0500 Subject: [PATCH 4/5] Update src/Appwrite/Platform/Tasks/GetMigrationStats.php Co-authored-by: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> --- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index 6fddf2d7e0..b76e0428d7 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -114,13 +114,14 @@ class GetMigrationStats extends Action ]); $migrations = array_map(function ($migration) use ($project) { - $result['Project ID'] = $project->getId(); - $result['$id'] = $migration->getAttribute('$id'); - $result['$createdAt'] = $migration->getAttribute('$createdAt'); - $result['status'] = $migration->getAttribute('status'); - $result['stage'] = $migration->getAttribute('stage'); - $result['source'] = $migration->getAttribute('source'); - return array_values($result); + return [ + $project->getId(), + $migration->getAttribute('$id'), + $migration->getAttribute('$createdAt'), + $migration->getAttribute('status'), + $migration->getAttribute('stage'), + $migration->getAttribute('source'), + ]; }, $migrations); if (!empty($migrations)) { From 8eb8351d5ff74cfa71bdb9d021a2ed953f6e8496 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 20:26:38 -0500 Subject: [PATCH 5/5] feat: add migration counts to hamster --- src/Appwrite/Platform/Tasks/CalcTierStats.php | 8 ++++++++ src/Appwrite/Platform/Tasks/Hamster.php | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index ef900aab34..e6559a05d7 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -41,6 +41,7 @@ class CalcTierStats extends Action 'Functions', 'Deployments', 'Executions', + 'Migrations', ]; protected string $directory = '/usr/local'; @@ -290,6 +291,13 @@ class CalcTierStats extends Action $stats['Executions'] = 0; } + /** Get Total Migrations */ + try { + $stats['Migrations'] = $dbForProject->count('migrations', []); + } catch (\Throwable) { + $stats['Migrations'] = 0; + } + $csv->insertOne(array_values($stats)); } catch (\Throwable $th) { Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 1d5d3b0b26..2e2c2d466e 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -149,6 +149,9 @@ class Hamster extends Action /** Get Total Teams */ $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + /** Get Total Migrations */ + $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); + /** Get Total Members */ $teamInternalId = $project->getAttribute('teamInternalId', null); if ($teamInternalId) {