diff --git a/app/cli.php b/app/cli.php index 0b2cb884e6..ce978c6d9d 100644 --- a/app/cli.php +++ b/app/cli.php @@ -99,6 +99,10 @@ CLI::setResource('dbForPlatform', function ($pools, $cache) { return $dbForPlatform; }, ['pools', 'cache']); +CLI::setResource('console', function () { + return new Document(Config::getParam('console')); +}, []); + CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools diff --git a/app/config/console.php b/app/config/console.php new file mode 100644 index 0000000000..5c15a7930f --- /dev/null +++ b/app/config/console.php @@ -0,0 +1,52 @@ + ID::custom('console'), + '$internalId' => ID::custom('console'), + 'name' => 'Appwrite', + '$collection' => ID::custom('projects'), + 'description' => 'Appwrite core engine', + 'logo' => '', + 'teamId' => null, + 'webhooks' => [], + 'keys' => [], + 'platforms' => [ + [ + '$collection' => ID::custom('platforms'), + 'name' => 'Localhost', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => 'localhost', + ], // Current host is added on app init + ], + 'legalName' => '', + 'legalCountry' => '', + 'legalState' => '', + 'legalCity' => '', + 'legalAddress' => '', + 'legalTaxId' => '', + 'auths' => [ + 'mockNumbers' => [], + 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', + 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' + ], + 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], + 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], + 'oAuthProviders' => [ + 'githubEnabled' => true, + 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), + 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') + ], +]; + +return $console; diff --git a/app/controllers/general.php b/app/controllers/general.php index 9c7a07e970..1ac94b31bd 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -870,11 +870,9 @@ App::error() $publish = $error->getCode() === 0 || $error->getCode() >= 500; } - if ($error->getCode() >= 400 && $error->getCode() < 500) { + $providerConfig = System::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', ''); + if (!empty($providerConfig) && $error->getCode() >= 400 && $error->getCode() < 500) { // Register error logger - $providerName = System::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', ''); - $providerConfig = System::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', ''); - try { $loggingProvider = new DSN($providerConfig ?? ''); $providerName = $loggingProvider->getScheme(); diff --git a/app/init.php b/app/init.php index 554bed41d0..bc532da7ef 100644 --- a/app/init.php +++ b/app/init.php @@ -347,6 +347,7 @@ Config::load('apis', __DIR__ . '/config/apis.php'); // List of APIs Config::load('errors', __DIR__ . '/config/errors.php'); Config::load('oAuthProviders', __DIR__ . '/config/oAuthProviders.php'); Config::load('platforms', __DIR__ . '/config/platforms.php'); +Config::load('console', __DIR__ . '/config/console.php'); Config::load('collections', __DIR__ . '/config/collections.php'); Config::load('runtimes', __DIR__ . '/config/runtimes.php'); Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); @@ -817,6 +818,10 @@ $register->set('logger', function () { $providerName = System::getEnv('_APP_LOGGING_PROVIDER', ''); $providerConfig = System::getEnv('_APP_LOGGING_CONFIG', ''); + if (empty($providerConfig)) { + return; + } + try { $loggingProvider = new DSN($providerConfig ?? ''); @@ -1415,46 +1420,7 @@ App::setResource('session', function (Document $user) { }, ['user']); App::setResource('console', function () { - return new Document([ - '$id' => ID::custom('console'), - '$internalId' => ID::custom('console'), - 'name' => 'Appwrite', - '$collection' => ID::custom('projects'), - 'description' => 'Appwrite core engine', - 'logo' => '', - 'teamId' => null, - 'webhooks' => [], - 'keys' => [], - 'platforms' => [ - [ - '$collection' => ID::custom('platforms'), - 'name' => 'Localhost', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => 'localhost', - ], // Current host is added on app init - ], - 'legalName' => '', - 'legalCountry' => '', - 'legalState' => '', - 'legalCity' => '', - 'legalAddress' => '', - 'legalTaxId' => '', - 'auths' => [ - 'mockNumbers' => [], - 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', - 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' - ], - 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], - 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], - 'oAuthProviders' => [ - 'githubEnabled' => true, - 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), - 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') - ], - 'region' => System::getEnv('_APP_REGION', 'default') - ]); + return new Document(Config::getParam('console')); }, []); App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 2d37bdbf70..98a3f4d295 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -24,12 +24,13 @@ class Maintenance extends Action $this ->desc('Schedules maintenance tasks and publishes them to our queues') ->inject('dbForPlatform') + ->inject('console') ->inject('queueForCertificates') ->inject('queueForDeletes') - ->callback(fn (Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $queueForCertificates, $queueForDeletes)); + ->callback(fn (Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $console, $queueForCertificates, $queueForDeletes)); } - public function action(Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes): void + public function action(Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes): void { Console::title('Maintenance V1'); Console::success(APP_NAME . ' maintenance process v1 has started'); @@ -41,7 +42,7 @@ class Maintenance extends Action $cacheRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $schedulesDeletionRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day - Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $queueForDeletes, $queueForCertificates) { + Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $console, $queueForDeletes, $queueForCertificates) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); @@ -52,9 +53,14 @@ class Maintenance extends Action ->setProject($project) ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); - }); + $queueForDeletes + ->setType(DELETE_TYPE_MAINTENANCE) + ->setProject($console) + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) + ->trigger(); + $this->notifyDeleteConnections($queueForDeletes); $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 269b29c4c8..ed5ff8010a 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -7,7 +7,6 @@ use Exception; use Throwable; use Utopia\Audit\Audit; use Utopia\CLI\Console; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; @@ -102,7 +101,7 @@ class Audits extends Action 'mode' => $mode, 'data' => $auditPayload, ], - 'timestamp' => DateTime::formatTz(DateTime::now()) + 'timestamp' => date("Y-m-d H:i:s", $message->getTimestamp()), ]; if (isset($this->logs[$project->getInternalId()])) { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e11181d199..643fc4d010 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -47,7 +47,7 @@ class Deletes extends Action ->inject('project') ->inject('dbForPlatform') ->inject('getProjectDB') - ->inject('timelimit') + ->inject('getLogsDB') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -57,8 +57,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $project, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -131,7 +131,7 @@ class Deletes extends Action $this->deleteExpiredSessions($project, $getProjectDB); break; case DELETE_TYPE_USAGE: - $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); + $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime); break; case DELETE_TYPE_CACHE_BY_RESOURCE: $this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType); @@ -158,7 +158,7 @@ class Deletes extends Action $this->deleteExpiredTargets($project, $getProjectDB); $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); $this->deleteAuditLogs($project, $getProjectDB, $auditRetention); - $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); + $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime); $this->deleteExpiredSessions($project, $getProjectDB); break; default: @@ -412,14 +412,22 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void + private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { $dbForProject = $getProjectDB($project); - // Delete Usage stats + $dbForLogs = $getLogsDB($project); + + // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']), ], $dbForProject); + + // Delete Usage stats from logsDB + $this->deleteByGroup('stats', [ + Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::equal('period', ['1h']), + ], $dbForLogs); } /** @@ -851,7 +859,7 @@ class Deletes extends Action } else { Console::error('Failed to delete deployment files: ' . $deploymentPath); } - } catch (\Throwable $th) { + } catch (Throwable $th) { Console::error('Failed to delete deployment files: ' . $deploymentPath); Console::error('[Error] Type: ' . get_class($th)); Console::error('[Error] Message: ' . $th->getMessage()); @@ -881,7 +889,7 @@ class Deletes extends Action } else { Console::error('Failed to delete build files: ' . $buildPath); } - } catch (\Throwable $th) { + } catch (Throwable $th) { Console::error('Failed to delete deployment files: ' . $buildPath); Console::error('[Error] Type: ' . get_class($th)); Console::error('[Error] Message: ' . $th->getMessage()); @@ -947,7 +955,7 @@ class Deletes extends Action try { $documents = $database->deleteDocuments($collection, $queries); - } catch (\Throwable $th) { + } catch (Throwable $th) { Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage()); return; }