From a63667f5a3dfe741e89deac64a2e1437d43cbd40 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 May 2024 07:04:40 +0000 Subject: [PATCH 1/4] chore: cleanup obsolete tasks --- Dockerfile | 2 - bin/calc-tier-stats | 3 - bin/calc-users-stats | 3 - bin/clear-card-cache | 3 - bin/create-inf-metric | 3 - bin/delete-orphaned-projects | 3 - bin/dev-generate-translations | 3 - bin/get-migration-stats | 3 - bin/hamster | 3 - bin/patch-delete-project-collections | 3 - ...patch-delete-schedule-updated-at-attribute | 3 - bin/patch-recreate-repositories-documents | 3 - bin/volume-sync | 3 - src/Appwrite/Platform/Services/Tasks.php | 16 - src/Appwrite/Platform/Tasks/CalcTierStats.php | 407 --------------- .../Platform/Tasks/CreateInfMetric.php | 409 --------------- .../Platform/Tasks/DeleteOrphanedProjects.php | 161 ------ .../Tasks/DevGenerateTranslations.php | 136 ----- .../Platform/Tasks/GetMigrationStats.php | 187 ------- src/Appwrite/Platform/Tasks/Hamster.php | 157 ------ .../PatchRecreateRepositoriesDocuments.php | 169 ------ src/Appwrite/Platform/Tasks/VolumeSync.php | 59 --- src/Appwrite/Platform/Workers/Hamster.php | 494 ------------------ 23 files changed, 2233 deletions(-) delete mode 100644 bin/calc-tier-stats delete mode 100644 bin/calc-users-stats delete mode 100644 bin/clear-card-cache delete mode 100644 bin/create-inf-metric delete mode 100644 bin/delete-orphaned-projects delete mode 100755 bin/dev-generate-translations delete mode 100644 bin/get-migration-stats delete mode 100644 bin/hamster delete mode 100644 bin/patch-delete-project-collections delete mode 100644 bin/patch-delete-schedule-updated-at-attribute delete mode 100644 bin/patch-recreate-repositories-documents delete mode 100644 bin/volume-sync delete mode 100644 src/Appwrite/Platform/Tasks/CalcTierStats.php delete mode 100644 src/Appwrite/Platform/Tasks/CreateInfMetric.php delete mode 100644 src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php delete mode 100644 src/Appwrite/Platform/Tasks/DevGenerateTranslations.php delete mode 100644 src/Appwrite/Platform/Tasks/GetMigrationStats.php delete mode 100644 src/Appwrite/Platform/Tasks/Hamster.php delete mode 100644 src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php delete mode 100644 src/Appwrite/Platform/Tasks/VolumeSync.php delete mode 100644 src/Appwrite/Platform/Workers/Hamster.php diff --git a/Dockerfile b/Dockerfile index d7343dd71d..ee77ec956f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -99,12 +99,10 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-databases && \ chmod +x /usr/local/bin/worker-deletes && \ chmod +x /usr/local/bin/worker-functions && \ - chmod +x /usr/local/bin/worker-hamster && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ - chmod +x /usr/local/bin/worker-hamster && \ chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-usage-dump diff --git a/bin/calc-tier-stats b/bin/calc-tier-stats deleted file mode 100644 index c7fb71e6fd..0000000000 --- a/bin/calc-tier-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php calc-tier-stats $@ \ No newline at end of file diff --git a/bin/calc-users-stats b/bin/calc-users-stats deleted file mode 100644 index 60472ed7a2..0000000000 --- a/bin/calc-users-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php calc-users-stats $@ \ No newline at end of file diff --git a/bin/clear-card-cache b/bin/clear-card-cache deleted file mode 100644 index b39bc13651..0000000000 --- a/bin/clear-card-cache +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php clear-card-cache $@ \ No newline at end of file diff --git a/bin/create-inf-metric b/bin/create-inf-metric deleted file mode 100644 index ea7b2b4da0..0000000000 --- a/bin/create-inf-metric +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php create-inf-metric $@ \ No newline at end of file diff --git a/bin/delete-orphaned-projects b/bin/delete-orphaned-projects deleted file mode 100644 index 16b9e3184d..0000000000 --- a/bin/delete-orphaned-projects +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php delete-orphaned-projects $@ \ No newline at end of file diff --git a/bin/dev-generate-translations b/bin/dev-generate-translations deleted file mode 100755 index 2339be8996..0000000000 --- a/bin/dev-generate-translations +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php dev-generate-translations $@ \ No newline at end of file diff --git a/bin/get-migration-stats b/bin/get-migration-stats deleted file mode 100644 index efc1934e15..0000000000 --- a/bin/get-migration-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php get-migration-stats $@ \ No newline at end of file diff --git a/bin/hamster b/bin/hamster deleted file mode 100644 index dcc7ed308d..0000000000 --- a/bin/hamster +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php hamster $@ \ No newline at end of file diff --git a/bin/patch-delete-project-collections b/bin/patch-delete-project-collections deleted file mode 100644 index 8bf0736cf3..0000000000 --- a/bin/patch-delete-project-collections +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-delete-project-collections $@ \ No newline at end of file diff --git a/bin/patch-delete-schedule-updated-at-attribute b/bin/patch-delete-schedule-updated-at-attribute deleted file mode 100644 index 3e28289cbe..0000000000 --- a/bin/patch-delete-schedule-updated-at-attribute +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@ \ No newline at end of file diff --git a/bin/patch-recreate-repositories-documents b/bin/patch-recreate-repositories-documents deleted file mode 100644 index 8c6c4157f4..0000000000 --- a/bin/patch-recreate-repositories-documents +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@ \ No newline at end of file diff --git a/bin/volume-sync b/bin/volume-sync deleted file mode 100644 index 5190750a24..0000000000 --- a/bin/volume-sync +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php volume-sync $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index f1fd1945c4..ac1f99eec3 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -2,17 +2,10 @@ namespace Appwrite\Platform\Services; -use Appwrite\Platform\Tasks\CalcTierStats; -use Appwrite\Platform\Tasks\CreateInfMetric; -use Appwrite\Platform\Tasks\DeleteOrphanedProjects; -use Appwrite\Platform\Tasks\DevGenerateTranslations; use Appwrite\Platform\Tasks\Doctor; -use Appwrite\Platform\Tasks\GetMigrationStats; -use Appwrite\Platform\Tasks\Hamster; use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; -use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; use Appwrite\Platform\Tasks\QueueCount; use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\ScheduleFunctions; @@ -23,7 +16,6 @@ use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; -use Appwrite\Platform\Tasks\VolumeSync; use Utopia\Platform\Service; class Tasks extends Service @@ -32,17 +24,10 @@ class Tasks extends Service { $this->type = self::TYPE_CLI; $this - ->addAction(CalcTierStats::getName(), new CalcTierStats()) - ->addAction(CreateInfMetric::getName(), new CreateInfMetric()) - ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) - ->addAction(DevGenerateTranslations::getName(), new DevGenerateTranslations()) ->addAction(Doctor::getName(), new Doctor()) - ->addAction(GetMigrationStats::getName(), new GetMigrationStats()) - ->addAction(Hamster::getName(), new Hamster()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) - ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) ->addAction(QueueCount::getName(), new QueueCount()) ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) @@ -53,7 +38,6 @@ class Tasks extends Service ->addAction(Upgrade::getName(), new Upgrade()) ->addAction(Vars::getName(), new Vars()) ->addAction(Version::getName(), new Version()) - ->addAction(VolumeSync::getName(), new VolumeSync()) ; } } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php deleted file mode 100644 index ae8bd27fe3..0000000000 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ /dev/null @@ -1,407 +0,0 @@ - 'Requests', - 'network.inbound' => 'Inbound', - 'network.outbound' => 'Outbound', - - ]; - - public static function getName(): string - { - return 'calc-tier-stats'; - } - - public function __construct() - { - - $this - ->desc('Get stats for projects') - ->param('after', '', new Text(36), 'After cursor', true) - ->param('projectId', '', new Text(36), 'Select project to validate', true) - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('getProjectDB') - ->inject('register') - ->callback(function ($after, $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) { - $this->action($after, $projectId, $pools, $cache, $dbForConsole, $getProjectDB, $register); - }); - } - - - public function action(string $after, string $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register): void - { - //docker compose exec -t appwrite calc-tier-stats - - Console::title('Cloud free tier stats calculation V1'); - Console::success(APP_NAME . ' cloud free tier stats calculation has started'); - - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/tier_stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); - - if (!empty($projectId)) { - try { - console::log("Project " . $projectId); - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $data = $this->getData($project, $dbForConsole, $dbForProject); - $csv->insertOne($data); - $this->sendMail($register); - - return; - } catch (\Throwable $th) { - Console::error("Unexpected error occurred with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - } - - $queries = []; - - if (!empty($after)) { - Console::info("Iterating remaining projects after project with ID {$after}"); - $project = $dbForConsole->getDocument('projects', $after); - $queries = [Query::cursorAfter($project)]; - } else { - Console::info("Iterating all projects"); - } - - $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole, $csv) { - $projectId = $project->getId(); - console::log("Project " . $projectId); - try { - $dbForProject = call_user_func($getProjectDB, $project); - $data = $this->getData($project, $dbForConsole, $dbForProject); - $csv->insertOne($data); - } catch (\Throwable $th) { - Console::error("Unexpected error occurred with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - }); - - $this->sendMail($register); - } - - private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void - { - $limit = 1000; - $results = []; - $sum = $limit; - $latestDocument = null; - - while ($sum === $limit) { - $newQueries = $queries; - - if ($latestDocument != null) { - array_unshift($newQueries, Query::cursorAfter($latestDocument)); - } - $newQueries[] = Query::limit($limit); - $results = $database->find('projects', $newQueries); - - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - private function sendMail(Registry $register): void - { - /** @var PHPMailer $mail */ - $mail = $register->get('smtp'); - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - - try { - /** Addresses */ - $mail->setFrom(System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); - $recipients = explode(',', System::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); - foreach ($recipients as $recipient) { - $mail->addAddress($recipient); - } - - /** Attachments */ - $mail->addAttachment($this->path); - - /** Content */ - $mail->Subject = "Cloud Report for {$this->date}"; - $mail->Body = "Please find the daily cloud report atttached"; - $mail->send(); - Console::success('Email has been sent!'); - } catch (\Throwable $e) { - Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - } - } - - - private function getData(Document $project, Database $dbForConsole, Database $dbForProject): array - { - $stats['Project ID'] = $project->getId(); - $stats['Organization ID'] = $project->getAttribute('teamId', null); - - $teamInternalId = $project->getAttribute('teamInternalId', null); - - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - Console::error('Membership not found. Skipping project : ' . $project->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $stats['Organization Email'] = $user->getAttribute('email', null); - } - } else { - Console::error("Email was not found for this Organization ID :{$teamInternalId}"); - } - - /** Get Total Members */ - if ($teamInternalId) { - $stats['Organization Members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ]); - } else { - $stats['Organization Members'] = 0; - } - - /** Get Total internal Teams */ - try { - $stats['Teams'] = $dbForProject->count('teams', []); - } catch (\Throwable) { - $stats['Teams'] = 0; - } - - /** Get Total users */ - try { - $stats['Users'] = $dbForProject->count('users', []); - } catch (\Throwable) { - $stats['Users'] = 0; - } - - /** Get Usage stats */ - $range = '30d'; - $periods = [ - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ] - ]; - - $tmp = []; - $metrics = $this->usageStats; - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) { - foreach ($metrics as $metric => $name) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', [$period]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $tmp[$metric] = []; - foreach ($requestDocs as $requestDoc) { - if (empty($requestDoc)) { - continue; - } - - $tmp[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $tmp[$metric] = array_reverse($tmp[$metric]); - $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value')); - } - }); - - foreach ($tmp as $key => $value) { - $stats[$metrics[$key]] = $value; - } - - /** - * Workaround to combine network.inbound+network.outbound as network. - */ - $stats['Bandwidth'] = ($stats['Inbound'] ?? 0) + ($stats['Outbound'] ?? 0); - unset($stats['Inbound']); - unset($stats['Outbound']); - - try { - /** Get Domains */ - $stats['Domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Domains'] = 0; - } - - try { - /** Get Api keys */ - $stats['Api keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Api keys'] = 0; - } - - try { - /** Get Webhooks */ - $stats['Webhooks'] = $dbForConsole->count('webhooks', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Webhooks'] = 0; - } - - try { - /** Get Platforms */ - $stats['Platforms'] = $dbForConsole->count('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Platforms'] = 0; - } - - /** Get Files & Buckets */ - $filesCount = 0; - $filesSum = 0; - $maxFileSize = 0; - $counter = 0; - try { - $buckets = $dbForProject->find('buckets', []); - foreach ($buckets as $bucket) { - $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]); - if (empty($file)) { - continue; - } - $filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []); - $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []); - if ($file->getAttribute('sizeOriginal') > $maxFileSize) { - $maxFileSize = $file->getAttribute('sizeOriginal'); - } - $counter++; - } - } catch (\Throwable $t) { - Console::error("Error while counting buckets: {$project->getId()}"); - } - $stats['Buckets'] = $counter; - $stats['Files'] = $filesCount; - $stats['Storage (bytes)'] = $filesSum; - $stats['Max File Size (bytes)'] = $maxFileSize; - - - try { - /** Get Total Functions */ - $stats['Databases'] = $dbForProject->count('databases', []); - } catch (\Throwable) { - $stats['Databases'] = 0; - } - - /** Get Total Functions */ - try { - $stats['Functions'] = $dbForProject->count('functions', []); - } catch (\Throwable) { - $stats['Functions'] = 0; - } - - /** Get Total Deployments */ - try { - $stats['Deployments'] = $dbForProject->count('deployments', []); - } catch (\Throwable) { - $stats['Deployments'] = 0; - } - - /** Get Total Executions */ - try { - $stats['Executions'] = $dbForProject->count('executions', []); - } catch (\Throwable) { - $stats['Executions'] = 0; - } - - /** Get Total Migrations */ - try { - $stats['Migrations'] = $dbForProject->count('migrations', []); - } catch (\Throwable) { - $stats['Migrations'] = 0; - } - - return array_values($stats); - } -} diff --git a/src/Appwrite/Platform/Tasks/CreateInfMetric.php b/src/Appwrite/Platform/Tasks/CreateInfMetric.php deleted file mode 100644 index ce3484edbf..0000000000 --- a/src/Appwrite/Platform/Tasks/CreateInfMetric.php +++ /dev/null @@ -1,409 +0,0 @@ -desc('Create infinity stats metric') - ->param('after', '', new Text(36), 'After cursor', true) - ->param('projectId', '', new Text(36), 'Select project to validate', true) - ->inject('getProjectDB') - ->inject('dbForConsole') - ->callback(function (string $after, string $projectId, callable $getProjectDB, Database $dbForConsole) { - $this->action($after, $projectId, $getProjectDB, $dbForConsole); - }); - } - - - /** - * @throws Exception - * @throws Exception\Timeout - * @throws Exception\Query - */ - public function action(string $after, string $projectId, callable $getProjectDB, Database $dbForConsole): void - { - - Console::title('Create infinity metric V1'); - Console::success(APP_NAME . ' Create infinity metric started'); - - if (!empty($projectId)) { - try { - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $this->getUsageData($dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occurred with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - } else { - $queries = []; - if (!empty($after)) { - Console::info("Iterating remaining projects after project with ID {$after}"); - $project = $dbForConsole->getDocument('projects', $after); - $queries = [Query::cursorAfter($project)]; - } else { - Console::info("Iterating all projects"); - } - $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB) { - $projectId = $project->getId(); - - try { - $dbForProject = call_user_func($getProjectDB, $project); - $this->getUsageData($dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occurred with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - }); - } - } - - /** - * @param Database $database - * @param string $collection - * @param array $queries - * @param callable|null $callback - * @return void - * @throws Exception - * @throws Exception\Query - * @throws Exception\Timeout - */ - private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void - { - $limit = 1000; - $results = []; - $sum = $limit; - $latestDocument = null; - - while ($sum === $limit) { - $newQueries = $queries; - - if ($latestDocument != null) { - array_unshift($newQueries, Query::cursorAfter($latestDocument)); - } - $newQueries[] = Query::limit($limit); - $results = $database->find($collection, $newQueries); - - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - - /** - * @param Database $dbForProject - * @param Document $project - * @return void - */ - private function getUsageData(Database $dbForProject, Document $project): void - { - try { - $this->network($dbForProject); - $this->sessions($dbForProject); - $this->users($dbForProject); - $this->teams($dbForProject); - $this->databases($dbForProject); - $this->functions($dbForProject); - $this->storage($dbForProject); - } catch (\Throwable $th) { - var_dump($th->getMessage()); - } - - Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId()); - } - - - /** - * @param Database $dbForProject - * @param string $metric - * @param int|float $value - * @return void - * @throws Exception - * @throws Exception\Authorization - * @throws Exception\Conflict - * @throws Exception\Restricted - * @throws Exception\Structure - */ - private function createInfMetric(database $dbForProject, string $metric, int|float $value): void - { - - try { - $id = \md5("_inf_{$metric}"); - $dbForProject->deleteDocument('stats', $id); - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'metric' => $metric, - 'period' => 'inf', - 'value' => (int)$value, - 'time' => null, - 'region' => 'default', - ])); - } catch (Duplicate $th) { - console::log("Error while creating inf metric: duplicate id {$metric} {$id}"); - } - } - - /** - * @param Database $dbForProject - * @param string $metric - * @return int|float - * @throws Exception - */ - protected function getFromMetric(database $dbForProject, string $metric): int|float - { - - return $dbForProject->sum('stats', 'value', [ - Query::equal('metric', [ - $metric, - ]), - Query::equal('period', ['1d']), - ]); - } - - /** - * @param Database $dbForProject - * @throws Exception - * @throws Exception\Authorization - * @throws Exception\Conflict - * @throws Exception\Restricted - * @throws Exception\Structure - */ - private function network(database $dbForProject) - { - $this->createInfMetric($dbForProject, 'network.inbound', $this->getFromMetric($dbForProject, 'network.inbound')); - $this->createInfMetric($dbForProject, 'network.outbound', $this->getFromMetric($dbForProject, 'network.outbound')); - $this->createInfMetric($dbForProject, 'network.requests', $this->getFromMetric($dbForProject, 'network.requests')); - } - - - /** - * @throws Exception\Authorization - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception\Timeout - * @throws Exception\Structure - * @throws Exception - * @throws Exception\Query - */ - private function storage(database $dbForProject) - { - $bucketsCount = 0; - $filesCount = 0; - $filesStorageSum = 0; - - $buckets = $dbForProject->find('buckets'); - foreach ($buckets as $bucket) { - $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); - $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $files); - - $filesStorage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal'); - $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStorage); - - $bucketsCount++; - $filesCount += $files; - $filesStorageSum += $filesStorage; - } - - $this->createInfMetric($dbForProject, 'buckets', $bucketsCount); - $this->createInfMetric($dbForProject, 'files', $filesCount); - $this->createInfMetric($dbForProject, 'files.storage', $filesStorageSum); - } - - - /** - * @throws Exception\Authorization - * @throws Exception\Timeout - * @throws Exception\Restricted - * @throws Exception\Structure - * @throws Exception\Conflict - * @throws Exception - * @throws Exception\Query - */ - private function functions(Database $dbForProject) - { - $functionsCount = 0; - $deploymentsCount = 0; - $buildsCount = 0; - $buildsStorageSum = 0; - $buildsComputeSum = 0; - $executionsCount = 0; - $executionsComputeSum = 0; - $deploymentsStorageSum = 0; - - //functions - $functions = $dbForProject->find('functions'); - foreach ($functions as $function) { - //deployments - $deployments = $dbForProject->find('deployments', [ - Query::equal('resourceType', ['functions']), - Query::equal('resourceInternalId', [$function->getInternalId()]), - ]); - - $deploymentCount = 0; - $deploymentStorageSum = 0; - foreach ($deployments as $deployment) { - //builds - $builds = $dbForProject->count('builds', [ - Query::equal('deploymentInternalId', [$deployment->getInternalId()]), - ]); - - $buildsCompute = $dbForProject->sum('builds', 'duration', [ - Query::equal('deploymentInternalId', [$deployment->getInternalId()]), - ]); - - $buildsStorage = $dbForProject->sum('builds', 'size', [ - Query::equal('deploymentInternalId', [$deployment->getInternalId()]), - ]); - - $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds', $builds); - $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.storage', $buildsCompute * 1000); - $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildsStorage); - - $buildsCount += $builds; - $buildsComputeSum += $buildsCompute; - $buildsStorageSum += $buildsStorage; - - - $deploymentCount++; - $deploymentsCount++; - $deploymentsStorageSum += $deployment['size']; - $deploymentStorageSum += $deployment['size']; - } - $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments', $deploymentCount); - $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments.storage', $deploymentStorageSum); - - //executions - $executions = $dbForProject->count('executions', [ - Query::equal('functionInternalId', [$function->getInternalId()]), - ]); - - $executionsCompute = $dbForProject->sum('executions', 'duration', [ - Query::equal('functionInternalId', [$function->getInternalId()]), - ]); - - $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executions); - $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', $executionsCompute * 1000); - $executionsCount += $executions; - $executionsComputeSum += $executionsCompute; - - $functionsCount++; - } - - $this->createInfMetric($dbForProject, 'functions', $functionsCount); - $this->createInfMetric($dbForProject, 'deployments', $deploymentsCount); - $this->createInfMetric($dbForProject, 'deployments.storage', $deploymentsStorageSum); - $this->createInfMetric($dbForProject, 'builds', $buildsCount); - $this->createInfMetric($dbForProject, 'builds.compute', $buildsComputeSum * 1000); - $this->createInfMetric($dbForProject, 'builds.storage', $buildsStorageSum); - $this->createInfMetric($dbForProject, 'executions', $executionsCount); - $this->createInfMetric($dbForProject, 'executions.compute', $executionsComputeSum * 1000); - } - - /** - * @throws Exception\Authorization - * @throws Exception\Timeout - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - * @throws Exception\Query - */ - private function databases(Database $dbForProject) - { - $databasesCount = 0; - $collectionsCount = 0; - $documentsCount = 0; - $databases = $dbForProject->find('databases'); - foreach ($databases as $database) { - $collectionCount = 0; - $collections = $dbForProject->find('database_' . $database->getInternalId()); - foreach ($collections as $collection) { - $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - $this->createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documents); - $documentsCount += $documents; - $collectionCount++; - $collectionsCount++; - } - $this->createInfMetric($dbForProject, $database->getInternalId() . '.collections', $collectionCount); - $this->createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCount); - $databasesCount++; - } - $this->createInfMetric($dbForProject, 'collections', $collectionsCount); - $this->createInfMetric($dbForProject, 'databases', $databasesCount); - $this->createInfMetric($dbForProject, 'documents', $documentsCount); - } - - - /** - * @throws Exception\Authorization - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - */ - private function users(Database $dbForProject) - { - $users = $dbForProject->count('users'); - $this->createInfMetric($dbForProject, 'users', $users); - } - - /** - * @throws Exception\Authorization - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - */ - private function sessions(Database $dbForProject) - { - $users = $dbForProject->count('sessions'); - $this->createInfMetric($dbForProject, 'sessions', $users); - } - - /** - * @throws Exception\Authorization - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - */ - private function teams(Database $dbForProject) - { - $teams = $dbForProject->count('teams'); - $this->createInfMetric($dbForProject, 'teams', $teams); - } -} diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php deleted file mode 100644 index 94879a4386..0000000000 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ /dev/null @@ -1,161 +0,0 @@ -desc('Delete orphaned projects') - ->param('commit', false, new Boolean(true), 'Commit project deletion', true) - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($commit, $pools, $cache, $dbForConsole, $register); - }); - } - - - public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - - Console::title('Delete orphaned projects V1'); - Console::success(APP_NAME . ' Delete orphaned projects started'); - - /** @var array $collections */ - $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; - - $collectionsConfig = array_merge([ - 'audit' => [ - '$id' => ID::custom('audit'), - '$collection' => Database::METADATA - ], - 'abuse' => [ - '$id' => ID::custom('abuse'), - '$collection' => Database::METADATA - ] - ], $collectionsConfig); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - $projects = [$console]; - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $orphans = 1; - $cnt = 0; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $collectionsCreated = 0; - $cnt++; - if ($dbForProject->exists($dbForProject->getDatabase(), Database::METADATA)) { - $collectionsCreated = $dbForProject->count(Database::METADATA); - } - - $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; - - if ($collectionsCreated >= count($collectionsConfig)) { - Console::log($msg . ' ignoring....'); - continue; - } - - Console::log($msg); - - if ($collectionsCreated > 0) { - $collections = $dbForProject->find(Database::METADATA, []); - foreach ($collections as $collection) { - if ($commit) { - $dbForProject->deleteCollection($collection->getId()); - $dbForConsole->purgeCachedCollection($collection->getId()); - } - Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); - } - } - - if ($commit) { - $dbForConsole->deleteDocument('projects', $project->getId()); - $dbForConsole->purgeCachedDocument('projects', $project->getId()); - - if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { - try { - $dbForProject->deleteCollection(Database::METADATA); - $dbForProject->purgeCachedCollection(Database::METADATA); - } catch (\Throwable $th) { - Console::warning('Metadata collection does not exist'); - } - } - } - - Console::info('--Deleting project no (' . $project->getInternalId() . ')'); - - $orphans++; - } catch (\Throwable $th) { - Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); - } -} diff --git a/src/Appwrite/Platform/Tasks/DevGenerateTranslations.php b/src/Appwrite/Platform/Tasks/DevGenerateTranslations.php deleted file mode 100644 index 66eee00d0d..0000000000 --- a/src/Appwrite/Platform/Tasks/DevGenerateTranslations.php +++ /dev/null @@ -1,136 +0,0 @@ -desc('Generate translations in all languages') - ->param('dry-run', 'true', new Boolean(true), 'If action should do a dry run. Dry run does not write into files', true) - ->param('api-key', '', new Text(256), 'Open AI API key. Only used during non-dry runs to generate translations.', true) - ->callback(fn ($dryRun, $apiKey) => $this->action($dryRun, $apiKey)); - } - - public function action(bool|string $dryRun, string $apiKey): void - { - $dryRun = \strval($dryRun) === 'true'; - - Console::info("Started"); - - if (!$dryRun && empty($apiKey)) { - Console::error("Please specify --api-key='OPEN_AI_API_KEY' or run with --dry-run"); - return; - } - - $this->apiKey = $apiKey; - - $dir = __DIR__ . '/../../../../app/config/locale/translations'; - $mainFile = 'en.json'; - - $mainJson = \json_decode(\file_get_contents($dir . '/' . $mainFile), true); - $mainKeys = \array_keys($mainJson); - - $files = array_diff(scandir($dir), array('.', '..', $mainFile)); - - foreach ($files as $file) { - $fileJson = \json_decode(\file_get_contents($dir . '/' . $file), true); - $fileKeys = \array_keys($fileJson); - - // Trick to clear specific key from all translation files: - // $json = \json_decode(\file_get_contents($dir . '/' . $file), true); - // unset($json['emails.magicSession.optionUrl']); - // \file_put_contents($dir . '/' . $file, \json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | 0)); - // continue; - - foreach ($mainKeys as $key) { - if (!(\in_array($key, $fileKeys))) { - if ($dryRun) { - Console::warning("{$file} missing translation for {$key}"); - } else { - $language = \explode('.', $file)[0]; - $translation = $this->generateTranslation($language, $mainJson[$key]); - - if (!empty($translation)) { - $json = \json_decode(\file_get_contents($dir . '/' . $file), true); - $json[$key] = $translation; - \file_put_contents($dir . '/' . $file, \json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | 0)); - - Console::success("Generated {$key} for {$language}"); - } - } - } - } - } - - Console::info("Done"); - } - - private function generateTranslation(string $targetLanguage, string $enTranslation): string - { - $list = Config::getParam('locale-languages'); - foreach ($list as $language) { - if ($language['code'] === $targetLanguage) { - $languageObject = $language; - } - } - - if (!isset($languageObject)) { - Console::error("{$targetLanguage} language not found"); - return ''; - } - - $targetLanguageName = $languageObject['name']; - - $response = Client::fetch('https://api.openai.com/v1/chat/completions', [ - 'content-type' => Client::CONTENT_TYPE_APPLICATION_JSON, - 'Authorization' => 'Bearer ' . $this->apiKey - ], Client::METHOD_POST, [ - 'model' => 'gpt-4-1106-preview', // https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo - 'messages' => [ - [ - 'role' => 'system', - 'content' => "Please translate the message user provides from English language to {$targetLanguageName}. Do not translate text inside {{ and }} placeholders. Provide only translated text." - ], - [ - 'role' => 'user', - 'content' => $enTranslation - ] - ] - ], [], 60); - - $body = \json_decode($response->getBody(), true); - - if ($response->getStatusCode() >= 400) { - throw new Exception($response->getBody() . ' with status code ' . $response->getStatusCode() . ' for language ' . $targetLanguage . ' and message ' . $enTranslation); - } - - $answer = $body['choices'][0]['message']['content']; - - $failureDetectors = [ 'sorry', 'confusion', 'country code', 'misunderstanding', 'correct', 'clarify', 'specific', 'cannot', 'unable', 'language', 'appears' ]; - - foreach ($failureDetectors as $detector) { - if (\str_contains($answer, $detector)) { - Console::error("Translation of '{$enTranslation}' for {$targetLanguage} is incorrect: {$answer}"); - } - } - - return $answer; - } -} diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php deleted file mode 100644 index 30a9ebb8e9..0000000000 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ /dev/null @@ -1,187 +0,0 @@ -desc('Get stats for projects') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); - }); - } - - /** - * @throws \Utopia\Exception - * @throws CannotInsertRecord - */ - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - //docker compose exec -t appwrite get-migration-stats - - Console::title('Migration stats calculation V1'); - Console::success(APP_NAME . ' Migration stats calculation has started'); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/migration_stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $projects = [$console]; - $count = 0; - $limit = 100; - $sum = 100; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - /** Get Project ID */ - $stats['Project ID'] = $project->getId(); - - /** Get Migration details */ - $migrations = $dbForProject->find('migrations', [ - Query::limit(500) - ]); - - $migrations = array_map(function ($migration) use ($project) { - return [ - $project->getId(), - $migration->getAttribute('$id'), - $migration->getAttribute('$createdAt'), - $migration->getAttribute('status'), - $migration->getAttribute('stage'), - $migration->getAttribute('source'), - ]; - }, $migrations); - - if (!empty($migrations)) { - $csv->insertAll($migrations); - } - } catch (\Throwable $th) { - Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); - - $pools - ->get('console') - ->reclaim(); - - /** @var PHPMailer $mail */ - $mail = $register->get('smtp'); - - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - - try { - /** Addresses */ - $mail->setFrom(System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); - $recipients = explode(',', System::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); - - foreach ($recipients as $recipient) { - $mail->addAddress($recipient); - } - - /** Attachments */ - $mail->addAttachment($this->path); - - /** Content */ - $mail->Subject = "Migration Report for {$this->date}"; - $mail->Body = "Please find the migration report atttached"; - $mail->send(); - Console::success('Email has been sent!'); - } catch (\Throwable $e) { - Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - } - } -} diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php deleted file mode 100644 index 2cfef1a2c9..0000000000 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ /dev/null @@ -1,157 +0,0 @@ -desc('Get stats for projects') - ->inject('queueForHamster') - ->inject('dbForConsole') - ->callback(function (EventHamster $queueForHamster, Database $dbForConsole) { - $this->action($queueForHamster, $dbForConsole); - }); - } - - public function action(EventHamster $queueForHamster, Database $dbForConsole): void - { - Console::title('Cloud Hamster V1'); - Console::success(APP_NAME . ' cloud hamster process has started'); - - $sleep = (int) System::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) - - $jobInitTime = System::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes) - - $now = new \DateTime(); - $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); - - $next = new \DateTime($now->format("Y-m-d $jobInitTime")); - $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); - - $delay = $next->getTimestamp() - $now->getTimestamp(); - /** - * If time passed for the target day. - */ - if ($delay <= 0) { - $next->add(\DateInterval::createFromDateString('1 days')); - $delay = $next->getTimestamp() - $now->getTimestamp(); - } - - Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']'); - - Console::loop(function () use ($queueForHamster, $dbForConsole, $sleep) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Queuing Cloud Usage Stats every {$sleep} seconds"); - $loopStart = microtime(true); - - Console::info('Queuing stats for all projects'); - $this->getStatsPerProject($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all projects'); - - Console::info('Queuing stats for all organizations'); - $this->getStatsPerOrganization($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all organizations'); - - Console::info('Queuing stats for all users'); - $this->getStatsPerUser($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all users'); - - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); - }, $sleep, $delay); - } - - protected function calculateByGroup(string $collection, Database $database, callable $callback) - { - $count = 0; - $chunk = 0; - $limit = 50; - $results = []; - $sum = $limit; - - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([ - Query::limit($limit), - Query::offset($count) - ])); - - $sum = count($results); - - Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents'); - - foreach ($results as $document) { - call_user_func($callback, $database, $document); - $count++; - } - } - - $executionEnd = \microtime(true); - - Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); - } - - protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster, $loopStart) { - try { - $organization->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_ORGANISATION) - ->setOrganization($organization) - ->trigger(); - } catch (\Throwable $e) { - Console::error($e->getMessage()); - } - }); - } - - private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster, $loopStart) { - try { - $project->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_PROJECT) - ->setProject($project) - ->trigger(); - } catch (\Throwable $e) { - Console::error($e->getMessage()); - } - }); - } - - protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster, $loopStart) { - try { - $user->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_USER) - ->setUser($user) - ->trigger(); - } catch (\Throwable $e) { - Console::error($e->getMessage()); - } - }); - } -} diff --git a/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php deleted file mode 100644 index a7e2367d59..0000000000 --- a/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php +++ /dev/null @@ -1,169 +0,0 @@ -desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.') - ->param('after', '', new Text(36), 'After cursor', true) - ->param('projectId', '', new Text(36), 'Select project to validate', true) - ->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB)); - } - - public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void - { - Console::info("Starting the patch"); - - $startTime = microtime(true); - - if (!empty($projectId)) { - try { - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occurred with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - } else { - $queries = []; - if (!empty($after)) { - Console::info("Iterating remaining projects after project with ID {$after}"); - $project = $dbForConsole->getDocument('projects', $after); - $queries = [Query::cursorAfter($project)]; - } else { - Console::info("Iterating all projects"); - } - $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) { - $projectId = $project->getId(); - - try { - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occurred with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - }); - } - - $endTime = microtime(true); - $timeTaken = $endTime - $startTime; - - $hours = (int)($timeTaken / 3600); - $timeTaken -= $hours * 3600; - $minutes = (int)($timeTaken / 60); - $timeTaken -= $minutes * 60; - $seconds = (int)$timeTaken; - $milliseconds = ($timeTaken - $seconds) * 1000; - Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)"); - } - - protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void - { - $limit = 1000; - $results = []; - $sum = $limit; - $latestDocument = null; - - while ($sum === $limit) { - $newQueries = $queries; - - if ($latestDocument != null) { - array_unshift($newQueries, Query::cursorAfter($latestDocument)); - } - $newQueries[] = Query::limit($limit); - $results = $database->find($collection, $newQueries); - - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void - { - $projectId = $project->getId(); - Console::log("Running patch for project {$projectId}"); - - $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) { - $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); - - if ($isConnected) { - $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); - - if ($repository->isEmpty()) { - $projectId = $project->getId(); - $functionId = $function->getId(); - Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); - - $repository = $dbForConsole->createDocument('repositories', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'installationId' => $function->getAttribute('installationId', ''), - 'installationInternalId' => $function->getAttribute('installationInternalId', ''), - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getInternalId(), - 'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''), - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getInternalId(), - 'resourceType' => 'function', - 'providerPullRequestIds' => [] - ])); - - $function = $dbForProject->updateDocument('functions', $function->getId(), $function - ->setAttribute('repositoryId', $repository->getId()) - ->setAttribute('repositoryInternalId', $repository->getInternalId())); - - $this->foreachDocument($dbForProject, 'deployments', [ - Query::equal('resourceInternalId', [$function->getInternalId()]), - Query::equal('resourceType', ['functions']) - ], function (Document $deployment) use ($dbForProject, $repository) { - $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment - ->setAttribute('repositoryId', $repository->getId()) - ->setAttribute('repositoryInternalId', $repository->getInternalId())); - }); - } - } - }); - } -} diff --git a/src/Appwrite/Platform/Tasks/VolumeSync.php b/src/Appwrite/Platform/Tasks/VolumeSync.php deleted file mode 100644 index 6197b20fbd..0000000000 --- a/src/Appwrite/Platform/Tasks/VolumeSync.php +++ /dev/null @@ -1,59 +0,0 @@ -desc('Runs rsync to sync certificates between the storage mount and traefik.') - ->param('source', null, new Text(255), 'Source path to sync from.', false) - ->param('destination', null, new Text(255), 'Destination path to sync to.', false) - ->param('interval', null, new Integer(true), 'Interval to run rsync', false) - ->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval)); - } - - public function action(string $source, string $destination, int $interval) - { - - Console::title('RSync V1'); - Console::success(APP_NAME . ' rsync process v1 has started'); - - if (!file_exists($source)) { - Console::error('Source directory does not exist. Exiting ... '); - Console::exit(0); - } - - Console::loop(function () use ($interval, $source, $destination) { - $time = DateTime::now(); - - Console::info("[{$time}] Executing rsync every {$interval} seconds"); - Console::info("Syncing between $source and $destination"); - - if (!file_exists($source)) { - Console::error('Source directory does not exist. Skipping ... '); - return; - } - - $stdin = ""; - $stdout = ""; - $stderr = ""; - - Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr); - Console::success($stdout); - Console::error($stderr); - }, $interval); - } -} diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php deleted file mode 100644 index 2c34f6f3dc..0000000000 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ /dev/null @@ -1,494 +0,0 @@ - 'files', - 'usage_buckets' => 'buckets', - 'usage_databases' => 'databases', - 'usage_documents' => 'documents', - 'usage_collections' => 'collections', - 'usage_storage' => 'files.storage', - 'usage_requests' => 'network.requests', - 'usage_inbound' => 'network.inbound', - 'usage_outbound' => 'network.outbound', - 'usage_users' => 'users', - 'usage_sessions' => 'sessions', - 'usage_executions' => 'executions', - ]; - - protected Mixpanel $mixpanel; - - public static function getName(): string - { - return 'hamster'; - } - - /** - * @throws \Exception - */ - public function __construct() - { - $this - ->desc('Hamster worker') - ->inject('message') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole) => $this->action($message, $pools, $cache, $dbForConsole)); - } - - /** - * @param Message $message - * @param Group $pools - * @param Cache $cache - * @param Database $dbForConsole - * - * @return void - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole): void - { - $token = System::getEnv('_APP_MIXPANEL_TOKEN', ''); - if (empty($token)) { - throw new \Exception('Missing MixPanel Token'); - } - $this->mixpanel = new Mixpanel($token); - - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new \Exception('Missing payload'); - } - - $type = $payload['type'] ?? ''; - - switch ($type) { - case EventHamster::TYPE_PROJECT: - $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole); - break; - case EventHamster::TYPE_ORGANISATION: - $this->getStatsForOrganization(new Document($payload['organization']), $dbForConsole); - break; - case EventHamster::TYPE_USER: - $this->getStatsPerUser(new Document($payload['user']), $dbForConsole); - break; - } - } - - /** - * @param Document $project - * @param Group $pools - * @param Cache $cache - * @param Database $dbForConsole - * @throws \Utopia\Database\Exception - */ - private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole): void - { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - Console::info("Skipping project console"); - return; - } - - Console::log("Getting stats for Project {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $statsPerProject = []; - - $statsPerProject['time'] = $project->getAttribute('$time'); - - /** Get Project ID */ - $statsPerProject['project_id'] = $project->getId(); - - /** Get project created time */ - $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); - - /** Get Project Name */ - $statsPerProject['project_name'] = $project->getAttribute('name'); - - /** Total Project Variables */ - $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); - - /** Total Migrations */ - $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); - - /** Get Custom SMTP */ - $smtp = $project->getAttribute('smtp', null); - if ($smtp) { - $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; - - /** Get Custom Templates Count */ - $templates = array_keys($project->getAttribute('templates', [])); - $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'email'); - }); - $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'sms'); - }); - } - - /** Get total relationship attributes */ - $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ - Query::equal('type', ['relationship']) - ], APP_LIMIT_COUNT); - - /** Get Total Functions */ - $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - - foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { - $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ - Query::equal('runtime', [$runtime]), - ], APP_LIMIT_COUNT); - } - - /** Get Total Deployments */ - $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ - Query::equal('type', ['manual']) - ], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ - Query::equal('type', ['vcs']) - ], APP_LIMIT_COUNT); - - /** Get VCS repos connected */ - $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ], APP_LIMIT_COUNT); - - /** Get Total Teams */ - $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ], APP_LIMIT_COUNT); - } else { - $statsPerProject['custom_organization_members'] = 0; - } - - /** Get Email and Name of the project owner */ - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new \Exception('Membership not found. Skipping project : ' . $project->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerProject['email'] = $user->getAttribute('email', null); - $statsPerProject['name'] = $user->getAttribute('name', null); - } - } - - /** Add billing information to the project */ - $organization = $dbForConsole->findOne('teams', [ - Query::equal('$internalId', [$teamInternalId]) - ]); - - $billing = $this->getBillingDetails($organization); - $statsPerProject['billing_plan'] = $billing['billing_plan'] ?? null; - $statsPerProject['billing_start_date'] = $billing['billing_start_date'] ?? null; - - /** Get Domains */ - $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Platforms */ - $platforms = $dbForConsole->find('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'web'; - })); - - $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'android'; - })); - - $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'apple'); - })); - - $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'flutter'); - })); - - $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; - - foreach ($flutterPlatforms as $flutterPlatform) { - $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { - return $platform['type'] === $flutterPlatform; - })); - } - - $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Usage $statsPerProject */ - $periods = [ - 'infinity' => [ - 'period' => '1d', - 'limit' => 90, - ], - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - ]; - - Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { - foreach ($this->metrics as $key => $metric) { - foreach ($periods as $periodKey => $periodValue) { - $limit = $periodValue['limit']; - $period = $periodValue['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $statsPerProject[$key . '_' . $periodKey] = []; - foreach ($requestDocs as $requestDoc) { - $statsPerProject[$key . '_' . $periodKey][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); - // Calculate aggregate of each metric - $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); - } - } - }); - - /** - * Workaround to combine network.Inbound+network.outbound as bandwidth. - */ - $statsPerProject["usage_bandwidth_infinity"] = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"]; - $statsPerProject["usage_bandwidth_24h"] = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"]; - unset($statsPerProject["usage_outbound_24h"]); - unset($statsPerProject["usage_inbound_24h"]); - unset($statsPerProject["usage_outbound_infinity"]); - unset($statsPerProject["usage_inbound_infinity"]); - - - if (isset($statsPerProject['email'])) { - /** Send data to mixpanel */ - $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ - 'name' => $statsPerProject['name'], - 'email' => $statsPerProject['email'] - ]); - - if (!$res) { - Console::error('Failed to create user profile for project: ' . $project->getId()); - } - } - - $event = new AnalyticsEvent(); - $event - ->setName('Project Daily Usage') - ->setProps($statsPerProject); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - Console::error('Failed to create event for project: ' . $project->getId()); - } - } catch (\Throwable $e) { - Console::error('Failed to send stats for project: ' . $project->getId()); - Console::error($e->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - /** - * @param Document $organization - * @param Database $dbForConsole - */ - private function getStatsForOrganization(Document $organization, Database $dbForConsole): void - { - Console::log("Getting stats for Organization {$organization->getId()}"); - - try { - $statsPerOrganization = []; - - $statsPerOrganization['time'] = $organization->getAttribute('$time'); - - /** Organization name */ - $statsPerOrganization['name'] = $organization->getAttribute('name'); - - /** Get Email and of the organization owner */ - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$organization->getInternalId()]), - ]); - if (!$membership || $membership->isEmpty()) { - throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId()); - } - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerOrganization['email'] = $user->getAttribute('email', null); - } - - /** Add billing information */ - $billing = $this->getBillingDetails($organization); - $statsPerOrganization['billing_plan'] = $billing['billing_plan'] ?? null; - $statsPerOrganization['billing_start_date'] = $billing['billing_start_date'] ?? null; - $statsPerOrganization['marked_for_deletion'] = $billing['markedForDeletion'] ?? 0; - $statsPerOrganization['billing_plan_downgrade'] = $billing['billing_plan_downgrade'] ?? null; - - /** Organization Creation Date */ - $statsPerOrganization['created'] = $organization->getAttribute('$createdAt'); - - /** Number of team members */ - $statsPerOrganization['members'] = $organization->getAttribute('total'); - - /** Number of projects in this organization */ - $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ - Query::equal('teamId', [$organization->getId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerOrganization['email'])) { - throw new \Exception('Email not found. Skipping organization : ' . $organization->getId()); - } - - $event = new AnalyticsEvent(); - $event - ->setName('Organization Daily Usage') - ->setProps($statsPerOrganization); - $res = $this->mixpanel->createEvent($event); - if (!$res) { - throw new \Exception('Failed to create event for organization : ' . $organization->getId()); - } - } catch (\Throwable $e) { - Console::error($e->getMessage()); - } - } - - protected function getStatsPerUser(Document $user, Database $dbForConsole) - { - Console::log("Getting stats for User {$user->getId()}"); - - try { - $statsPerUser = []; - - $statsPerUser['time'] = $user->getAttribute('$time'); - - /** Add billing information */ - $organization = $dbForConsole->findOne('teams', [ - Query::equal('userInternalId', [$user->getInternalId()]) - ]); - - - $billing = $this->getBillingDetails($organization); - $statsPerUser['billing_plan'] = $billing['billing_plan'] ?? null; - $statsPerUser['billing_start_date'] = $billing['billing_start_date'] ?? null; - - /** Organization name */ - $statsPerUser['name'] = $user->getAttribute('name'); - - /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ - $statsPerUser['email'] = $user->getAttribute('email'); - - /** Organization Creation Date */ - $statsPerUser['created'] = $user->getAttribute('$createdAt'); - - /** Number of teams this user is a part of */ - $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ - Query::equal('userInternalId', [$user->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerUser['email'])) { - throw new \Exception('User has no email: ' . $user->getId()); - } - - /** Send data to mixpanel */ - $event = new AnalyticsEvent(); - $event - ->setName('User Daily Usage') - ->setProps($statsPerUser); - - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - throw new \Exception('Failed to create user profile for user: ' . $user->getId()); - } - } catch (\Throwable $e) { - Console::error($e->getMessage()); - } - } - - private function getBillingDetails(bool|Document $team): array - { - $billing = []; - - if (!empty($team) && !$team->isEmpty()) { - $billingPlan = $team->getAttribute('billingPlan', null); - $billingPlanDowngrade = $team->getAttribute('billingPlanDowngrade', null); - - if (!empty($billingPlan) && empty($billingPlanDowngrade)) { - $billing['billing_plan'] = $billingPlan; - } - - if (in_array($billingPlan, ['tier-1', 'tier-2'])) { - $billingStartDate = $team->getAttribute('billingStartDate', null); - $billing['billing_start_date'] = $billingStartDate; - } - - $billing['marked_for_deletion'] = $team->getAttribute('markedForDeletion', 0); - $billing['billing_plan_downgrade'] = $billingPlanDowngrade; - } - - return $billing; - } -} From 10722aa736cb39478568a590eccfe3e995e1755a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 May 2024 07:11:00 +0000 Subject: [PATCH 2/4] chore: cleanup obsolete tasks and workers --- Dockerfile | 20 --- app/cli.php | 4 - app/controllers/api/health.php | 3 +- app/worker.php | 5 - bin/worker-hamster | 3 - docker-compose.yml | 57 -------- src/Appwrite/Event/Event.php | 3 - src/Appwrite/Event/Hamster.php | 157 --------------------- src/Appwrite/Platform/Services/Workers.php | 2 - src/Appwrite/Platform/Tasks/QueueCount.php | 3 +- src/Appwrite/Platform/Tasks/QueueRetry.php | 3 +- 11 files changed, 3 insertions(+), 257 deletions(-) delete mode 100644 bin/worker-hamster delete mode 100644 src/Appwrite/Event/Hamster.php diff --git a/Dockerfile b/Dockerfile index ee77ec956f..eedc759ed1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -106,26 +106,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-usage-dump - -# Cloud Executabless -RUN chmod +x /usr/local/bin/calc-tier-stats && \ - chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/clear-card-cache && \ - chmod +x /usr/local/bin/delete-orphaned-projects && \ - chmod +x /usr/local/bin/get-migration-stats && \ - chmod +x /usr/local/bin/hamster && \ - chmod +x /usr/local/bin/patch-delete-project-collections && \ - chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ - chmod +x /usr/local/bin/patch-recreate-repositories-documents && \ - chmod +x /usr/local/bin/volume-sync && \ - chmod +x /usr/local/bin/patch-delete-project-collections && \ - chmod +x /usr/local/bin/delete-orphaned-projects && \ - chmod +x /usr/local/bin/clear-card-cache && \ - chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats && \ - chmod +x /usr/local/bin/get-migration-stats && \ - chmod +x /usr/local/bin/create-inf-metric - # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/cli.php b/app/cli.php index bde8159f24..1996b4da32 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,7 +6,6 @@ require_once __DIR__ . '/controllers/general.php'; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; -use Appwrite\Event\Hamster; use Appwrite\Platform\Appwrite; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -131,9 +130,6 @@ CLI::setResource('queue', function (Group $pools) { CLI::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); -CLI::setResource('queueForHamster', function (Connection $queue) { - return new Hamster($queue); -}, ['queue']); CLI::setResource('queueForDeletes', function (Connection $queue) { return new Delete($queue); }, ['queue']); diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index c215a78a8b..4b4ce8f307 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -856,8 +856,7 @@ App::get('/v1/health/queue/failed/:name') Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, Event::MESSAGING_QUEUE_NAME, - Event::MIGRATIONS_QUEUE_NAME, - Event::HAMSTER_CLASS_NAME + Event::MIGRATIONS_QUEUE_NAME ]), 'The name of the queue') ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md') diff --git a/app/worker.php b/app/worker.php index ae9732fb1b..6f912a84fd 100644 --- a/app/worker.php +++ b/app/worker.php @@ -9,7 +9,6 @@ use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\Hamster; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; @@ -194,10 +193,6 @@ Server::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); -Server::setResource('queueForHamster', function (Connection $queue) { - return new Hamster($queue); -}, ['queue']); - Server::setResource('logger', function (Registry $register) { return $register->get('logger'); }, ['register']); diff --git a/bin/worker-hamster b/bin/worker-hamster deleted file mode 100644 index b388dd13c9..0000000000 --- a/bin/worker-hamster +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php hamster $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d270074412..3ef5f1b714 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -788,63 +788,6 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY - appwrite-worker-hamster: - entrypoint: worker-hamster - <<: *x-logging - container_name: appwrite-worker-hamster - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_MIXPANEL_TOKEN - - appwrite-hamster-scheduler: - entrypoint: hamster - <<: *x-logging - container_name: appwrite-hamster-scheduler - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_HAMSTER_TIME - - _APP_HAMSTER_INTERVAL - openruntimes-executor: container_name: openruntimes-executor hostname: appwrite-executor diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index fa9ddf1315..b80ba9333d 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -45,9 +45,6 @@ class Event public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; - public const HAMSTER_QUEUE_NAME = 'v1-hamster'; - public const HAMSTER_CLASS_NAME = 'HamsterV1'; - protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php deleted file mode 100644 index 5d79fce568..0000000000 --- a/src/Appwrite/Event/Hamster.php +++ /dev/null @@ -1,157 +0,0 @@ -setQueue(Event::HAMSTER_QUEUE_NAME) - ->setClass(Event::HAMSTER_CLASS_NAME); - } - - /** - * Sets the type for the hamster event. - * - * @param string $type - * @return self - */ - public function setType(string $type): self - { - $this->type = $type; - - return $this; - } - - /** - * Returns the set type for the hamster event. - * - * @return string - */ - public function getType(): string - { - return $this->type; - } - - /** - * Sets the project for the hamster event. - * - * @param Document $project - */ - public function setProject(Document $project): self - { - $this->project = $project; - - return $this; - } - - /** - * Returns the set project for the hamster event. - * - * @return Document - */ - public function getProject(): Document - { - return $this->project; - } - - /** - * Sets the organization for the hamster event. - * - * @param Document $organization - */ - public function setOrganization(Document $organization): self - { - $this->organization = $organization; - - return $this; - } - - /** - * Returns the set organization for the hamster event. - * - * @return string - */ - public function getOrganization(): Document - { - return $this->organization; - } - - /** - * Sets the user for the hamster event. - * - * @param Document $user - */ - public function setUser(Document $user): self - { - $this->user = $user; - - return $this; - } - - /** - * Returns the set user for the hamster event. - * - * @return Document - */ - public function getUser(): Document - { - return $this->user; - } - - /** - * Executes the function event and sends it to the functions worker. - * - * @return string|bool - * @throws \InvalidArgumentException - */ - public function trigger(): string|bool - { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; - - return $client->enqueue([ - 'type' => $this->type, - 'project' => $this->project, - 'organization' => $this->organization, - 'user' => $this->user, - 'events' => $events, - ]); - } - - /** - * Generate a function event from a base event - * - * @param Event $event - * - * @return self - * - */ - public function from(Event $event): self - { - $this->event = $event->getEvent(); - $this->params = $event->getParams(); - return $this; - } -} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 4adf9b76cb..62a7fcf3fb 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -8,7 +8,6 @@ use Appwrite\Platform\Workers\Certificates; use Appwrite\Platform\Workers\Databases; use Appwrite\Platform\Workers\Deletes; use Appwrite\Platform\Workers\Functions; -use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Mails; use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; @@ -32,7 +31,6 @@ class Workers extends Service ->addAction(Mails::getName(), new Mails()) ->addAction(Messaging::getName(), new Messaging()) ->addAction(Webhooks::getName(), new Webhooks()) - ->addAction(Hamster::getName(), new Hamster()) ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) diff --git a/src/Appwrite/Platform/Tasks/QueueCount.php b/src/Appwrite/Platform/Tasks/QueueCount.php index 9b7bba82d1..961360479d 100644 --- a/src/Appwrite/Platform/Tasks/QueueCount.php +++ b/src/Appwrite/Platform/Tasks/QueueCount.php @@ -32,8 +32,7 @@ class QueueCount extends Action Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, Event::MESSAGING_QUEUE_NAME, - Event::MIGRATIONS_QUEUE_NAME, - Event::HAMSTER_QUEUE_NAME + Event::MIGRATIONS_QUEUE_NAME ]), 'Queue name') ->param('type', '', new WhiteList([ 'success', diff --git a/src/Appwrite/Platform/Tasks/QueueRetry.php b/src/Appwrite/Platform/Tasks/QueueRetry.php index 465152f21d..7259f82c89 100644 --- a/src/Appwrite/Platform/Tasks/QueueRetry.php +++ b/src/Appwrite/Platform/Tasks/QueueRetry.php @@ -33,8 +33,7 @@ class QueueRetry extends Action Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, Event::MESSAGING_QUEUE_NAME, - Event::MIGRATIONS_QUEUE_NAME, - Event::HAMSTER_CLASS_NAME + Event::MIGRATIONS_QUEUE_NAME ]), 'Queue name') ->param('limit', 0, new Wildcard(), 'jobs limit', true) ->inject('queue') From b371909deb16d3b0a2b99cc3164a08884f5d59ae Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 May 2024 07:16:45 +0000 Subject: [PATCH 3/4] chore: fix Dockerile --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index eedc759ed1..1d2ac91ae0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -72,9 +72,6 @@ RUN mkdir -p /storage/uploads && \ chown -Rf www-data.www-data /storage/functions && chmod -Rf 0755 /storage/functions && \ chown -Rf www-data.www-data /storage/debug && chmod -Rf 0755 /storage/debug -# Development Executables -RUN chmod +x /usr/local/bin/dev-generate-translations - # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ From 82b473c8ee8b7291828c46178c985e47accdc144 Mon Sep 17 00:00:00 2001 From: bhc Date: Fri, 17 May 2024 17:39:35 +0300 Subject: [PATCH 4/4] Update Chinese language native names in languages.php --- app/config/locale/languages.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/locale/languages.php b/app/config/locale/languages.php index eeea92e636..28d3fa4ebc 100644 --- a/app/config/locale/languages.php +++ b/app/config/locale/languages.php @@ -926,12 +926,12 @@ return [ [ "code" => "zh-cn", "name" => "Chinese (Simplified)", - "nativeName" => "中国人" + "nativeName" => "中文" ], [ "code" => "zh-tw", "name" => "Chinese (Traditional)", - "nativeName" => "中國人" + "nativeName" => "中文" ], [ "code" => "zu",