From 6a6d99a9d0f87dabac610e7027b341ad0c9ac719 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 10 Mar 2025 08:54:20 +0000 Subject: [PATCH 1/2] chore: queue console project for maintenance delete --- app/cli.php | 45 +++++++++++++++++++++ src/Appwrite/Platform/Tasks/Maintenance.php | 14 +++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/app/cli.php b/app/cli.php index 0b2cb884e6..dbbdfdee4d 100644 --- a/app/cli.php +++ b/app/cli.php @@ -2,11 +2,13 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; use Appwrite\Event\StatsResources; use Appwrite\Event\StatsUsage; +use Appwrite\Network\Validator\Origin; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; use Utopia\Cache\Adapter\Sharding; @@ -16,6 +18,7 @@ use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Helpers\ID; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; @@ -99,6 +102,48 @@ CLI::setResource('dbForPlatform', function ($pools, $cache) { return $dbForPlatform; }, ['pools', 'cache']); +CLI::setResource('console', function () { + return new Document([ + '$id' => ID::custom('console'), + '$internalId' => ID::custom('console'), + 'name' => 'Appwrite', + '$collection' => ID::custom('projects'), + 'description' => 'Appwrite core engine', + 'logo' => '', + 'teamId' => null, + 'webhooks' => [], + 'keys' => [], + 'platforms' => [ + [ + '$collection' => ID::custom('platforms'), + 'name' => 'Localhost', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => 'localhost', + ], // Current host is added on app init + ], + 'legalName' => '', + 'legalCountry' => '', + 'legalState' => '', + 'legalCity' => '', + 'legalAddress' => '', + 'legalTaxId' => '', + 'auths' => [ + 'mockNumbers' => [], + 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', + 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' + ], + 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], + 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], + 'oAuthProviders' => [ + 'githubEnabled' => true, + 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), + 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') + ], + ]); +}, []); + CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 2d37bdbf70..98a3f4d295 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -24,12 +24,13 @@ class Maintenance extends Action $this ->desc('Schedules maintenance tasks and publishes them to our queues') ->inject('dbForPlatform') + ->inject('console') ->inject('queueForCertificates') ->inject('queueForDeletes') - ->callback(fn (Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $queueForCertificates, $queueForDeletes)); + ->callback(fn (Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $console, $queueForCertificates, $queueForDeletes)); } - public function action(Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes): void + public function action(Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes): void { Console::title('Maintenance V1'); Console::success(APP_NAME . ' maintenance process v1 has started'); @@ -41,7 +42,7 @@ class Maintenance extends Action $cacheRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $schedulesDeletionRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day - Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $queueForDeletes, $queueForCertificates) { + Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $console, $queueForDeletes, $queueForCertificates) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); @@ -52,9 +53,14 @@ class Maintenance extends Action ->setProject($project) ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); - }); + $queueForDeletes + ->setType(DELETE_TYPE_MAINTENANCE) + ->setProject($console) + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) + ->trigger(); + $this->notifyDeleteConnections($queueForDeletes); $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); From 61377ceb6e518d664f050337eafbde64e507e749 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 10 Mar 2025 10:48:44 +0000 Subject: [PATCH 2/2] chore: shift initialization of console project to config --- app/cli.php | 43 +--------------------------------- app/config/console.php | 52 ++++++++++++++++++++++++++++++++++++++++++ app/init.php | 41 ++------------------------------- 3 files changed, 55 insertions(+), 81 deletions(-) create mode 100644 app/config/console.php diff --git a/app/cli.php b/app/cli.php index dbbdfdee4d..ce978c6d9d 100644 --- a/app/cli.php +++ b/app/cli.php @@ -2,13 +2,11 @@ require_once __DIR__ . '/init.php'; -use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; use Appwrite\Event\StatsResources; use Appwrite\Event\StatsUsage; -use Appwrite\Network\Validator\Origin; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; use Utopia\Cache\Adapter\Sharding; @@ -18,7 +16,6 @@ use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Helpers\ID; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; @@ -103,45 +100,7 @@ CLI::setResource('dbForPlatform', function ($pools, $cache) { }, ['pools', 'cache']); CLI::setResource('console', function () { - return new Document([ - '$id' => ID::custom('console'), - '$internalId' => ID::custom('console'), - 'name' => 'Appwrite', - '$collection' => ID::custom('projects'), - 'description' => 'Appwrite core engine', - 'logo' => '', - 'teamId' => null, - 'webhooks' => [], - 'keys' => [], - 'platforms' => [ - [ - '$collection' => ID::custom('platforms'), - 'name' => 'Localhost', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => 'localhost', - ], // Current host is added on app init - ], - 'legalName' => '', - 'legalCountry' => '', - 'legalState' => '', - 'legalCity' => '', - 'legalAddress' => '', - 'legalTaxId' => '', - 'auths' => [ - 'mockNumbers' => [], - 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', - 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' - ], - 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], - 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], - 'oAuthProviders' => [ - 'githubEnabled' => true, - 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), - 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') - ], - ]); + return new Document(Config::getParam('console')); }, []); CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { diff --git a/app/config/console.php b/app/config/console.php new file mode 100644 index 0000000000..5c15a7930f --- /dev/null +++ b/app/config/console.php @@ -0,0 +1,52 @@ + ID::custom('console'), + '$internalId' => ID::custom('console'), + 'name' => 'Appwrite', + '$collection' => ID::custom('projects'), + 'description' => 'Appwrite core engine', + 'logo' => '', + 'teamId' => null, + 'webhooks' => [], + 'keys' => [], + 'platforms' => [ + [ + '$collection' => ID::custom('platforms'), + 'name' => 'Localhost', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => 'localhost', + ], // Current host is added on app init + ], + 'legalName' => '', + 'legalCountry' => '', + 'legalState' => '', + 'legalCity' => '', + 'legalAddress' => '', + 'legalTaxId' => '', + 'auths' => [ + 'mockNumbers' => [], + 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', + 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' + ], + 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], + 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], + 'oAuthProviders' => [ + 'githubEnabled' => true, + 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), + 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') + ], +]; + +return $console; diff --git a/app/init.php b/app/init.php index 1d7f734723..17c1a884f9 100644 --- a/app/init.php +++ b/app/init.php @@ -347,6 +347,7 @@ Config::load('apis', __DIR__ . '/config/apis.php'); // List of APIs Config::load('errors', __DIR__ . '/config/errors.php'); Config::load('oAuthProviders', __DIR__ . '/config/oAuthProviders.php'); Config::load('platforms', __DIR__ . '/config/platforms.php'); +Config::load('console', __DIR__ . '/config/console.php'); Config::load('collections', __DIR__ . '/config/collections.php'); Config::load('runtimes', __DIR__ . '/config/runtimes.php'); Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); @@ -1408,45 +1409,7 @@ App::setResource('session', function (Document $user) { }, ['user']); App::setResource('console', function () { - return new Document([ - '$id' => ID::custom('console'), - '$internalId' => ID::custom('console'), - 'name' => 'Appwrite', - '$collection' => ID::custom('projects'), - 'description' => 'Appwrite core engine', - 'logo' => '', - 'teamId' => null, - 'webhooks' => [], - 'keys' => [], - 'platforms' => [ - [ - '$collection' => ID::custom('platforms'), - 'name' => 'Localhost', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => 'localhost', - ], // Current host is added on app init - ], - 'legalName' => '', - 'legalCountry' => '', - 'legalState' => '', - 'legalCity' => '', - 'legalAddress' => '', - 'legalTaxId' => '', - 'auths' => [ - 'mockNumbers' => [], - 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', - 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' - ], - 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], - 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], - 'oAuthProviders' => [ - 'githubEnabled' => true, - 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), - 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') - ], - ]); + return new Document(Config::getParam('console')); }, []); App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) {