diff --git a/app/config/errors.php b/app/config/errors.php index c0628920d9..f0b857731b 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -680,6 +680,7 @@ return [ 'name' => Exception::RULE_VERIFICATION_FAILED, 'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.', 'code' => 401, + 'publish' => true ], Exception::PROJECT_SMTP_CONFIG_INVALID => [ 'name' => Exception::PROJECT_SMTP_CONFIG_INVALID, diff --git a/app/controllers/api/proxy.php b/app/controllers/api/proxy.php index 23916a114c..71125d2c87 100644 --- a/app/controllers/api/proxy.php +++ b/app/controllers/api/proxy.php @@ -14,6 +14,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; +use Utopia\Logger\Log; use Utopia\Validator\Domain as ValidatorDomain; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -278,7 +279,8 @@ App::patch('/v1/proxy/rules/:ruleId/verification') ->inject('queueForEvents') ->inject('project') ->inject('dbForConsole') - ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) { + ->inject('log') + ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole, Log $log) { $rule = $dbForConsole->getDocument('rules', $ruleId); if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) { @@ -298,7 +300,14 @@ App::patch('/v1/proxy/rules/:ruleId/verification') $validator = new CNAME($target->get()); // Verify Domain with DNS records $domain = new Domain($rule->getAttribute('domain', '')); + $validationStart = \microtime(true); if (!$validator->isValid($domain->get())) { + $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); + $log->addTag('dnsDomain', $domain->get()); + + $error = $validator->getLogs(); + $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); + throw new Exception(Exception::RULE_VERIFICATION_FAILED); } diff --git a/app/controllers/general.php b/app/controllers/general.php index e443b96fc9..2fce04b28e 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -603,9 +603,8 @@ App::error() ->inject('response') ->inject('project') ->inject('logger') - ->inject('loggerBreadcrumbs') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) { - + ->inject('log') + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $publish = true; @@ -614,55 +613,47 @@ App::error() $publish = $error->isPublishable(); } - if ($logger && $publish) { - if ($error->getCode() >= 500 || $error->getCode() === 0) { - try { - /** @var Utopia\Database\Document $user */ - $user = $utopia->getResource('user'); - } catch (\Throwable $th) { - // All good, user is optional information for logger - } - - $log = new Utopia\Logger\Log(); - - if (isset($user) && !$user->isEmpty()) { - $log->setUser(new User($user->getId())); - } - - $log->setNamespace("http"); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - $log->addTag('database', $project->getAttribute('database', 'console')); - $log->addTag('method', $route->getMethod()); - $log->addTag('url', $route->getPath()); - $log->addTag('verboseType', get_class($error)); - $log->addTag('code', $error->getCode()); - $log->addTag('projectId', $project->getId()); - $log->addTag('hostname', $request->getHostname()); - $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', ''))); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - $log->addExtra('roles', Authorization::getRoles()); - - $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); - $log->setAction($action); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - foreach ($loggerBreadcrumbs as $loggerBreadcrumb) { - $log->addBreadcrumb($loggerBreadcrumb); - } - - $responseCode = $logger->addLog($log); - Console::info('Log pushed with status code: ' . $responseCode); + if ($logger && ($publish || $error->getCode() === 0)) { + try { + /** @var Utopia\Database\Document $user */ + $user = $utopia->getResource('user'); + } catch (\Throwable $th) { + // All good, user is optional information for logger } + + if (isset($user) && !$user->isEmpty()) { + $log->setUser(new User($user->getId())); + } + + $log->setNamespace("http"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('database', $project->getAttribute('database', 'console')); + $log->addTag('method', $route->getMethod()); + $log->addTag('url', $route->getPath()); + $log->addTag('verboseType', get_class($error)); + $log->addTag('code', $error->getCode()); + $log->addTag('projectId', $project->getId()); + $log->addTag('hostname', $request->getHostname()); + $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', ''))); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + $log->addExtra('roles', Authorization::getRoles()); + + $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); + $log->setAction($action); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $responseCode = $logger->addLog($log); + Console::info('Log pushed with status code: ' . $responseCode); } $code = $error->getCode(); diff --git a/app/http.php b/app/http.php index fe1ed48724..5b32d8f134 100644 --- a/app/http.php +++ b/app/http.php @@ -263,10 +263,9 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo // All good, user is optional information for logger } - $loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs"); $route = $app->getRoute(); - $log = new Utopia\Logger\Log(); + $log = $app->getResource("log"); if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); @@ -298,10 +297,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - foreach ($loggerBreadcrumbs as $loggerBreadcrumb) { - $log->addBreadcrumb($loggerBreadcrumb); - } - $responseCode = $logger->addLog($log); Console::info('Log pushed with status code: ' . $responseCode); } diff --git a/app/init.php b/app/init.php index 0e9f16d6d6..a32ab08344 100644 --- a/app/init.php +++ b/app/init.php @@ -76,6 +76,7 @@ use Appwrite\Hooks\Hooks; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; +use Utopia\Logger\Log; use Utopia\Queue; use Utopia\Queue\Connection; use Utopia\Storage\Storage; @@ -864,6 +865,7 @@ foreach ($locales as $locale) { ]); // Runtime Execution +App::setResource('log', fn() => new Log()); App::setResource('logger', function ($register) { return $register->get('logger'); }, ['register']); @@ -872,10 +874,6 @@ App::setResource('hooks', function ($register) { return $register->get('hooks'); }, ['register']); -App::setResource('loggerBreadcrumbs', function () { - return []; -}); - App::setResource('register', fn() => $register); App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en'))); diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 6449ffd93a..195eedb353 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -238,21 +238,16 @@ class Exception extends \Exception protected string $type = ''; protected array $errors = []; - protected bool $publish = true; + protected bool $publish; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { $this->errors = Config::getParam('errors'); $this->type = $type; + $this->code = $code ?? $this->errors[$type]['code']; + $this->message = $message ?? $this->errors[$type]['description']; - if (isset($this->errors[$type])) { - $this->code = $this->errors[$type]['code']; - $this->message = $this->errors[$type]['description']; - $this->publish = $this->errors[$type]['publish'] ?? true; - } - - $this->message = $message ?? $this->message; - $this->code = $code ?? $this->code; + $this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500); parent::__construct($this->message, $this->code, $previous); } diff --git a/src/Appwrite/Network/Validator/CNAME.php b/src/Appwrite/Network/Validator/CNAME.php index 678a57cecd..e1ae061c84 100644 --- a/src/Appwrite/Network/Validator/CNAME.php +++ b/src/Appwrite/Network/Validator/CNAME.php @@ -6,6 +6,11 @@ use Utopia\Validator; class CNAME extends Validator { + /** + * @var mixed + */ + protected mixed $logs; + /** * @var string */ @@ -27,6 +32,14 @@ class CNAME extends Validator return 'Invalid CNAME record'; } + /** + * @return mixed + */ + public function getLogs(): mixed + { + return $this->logs; + } + /** * Check if CNAME record target value matches selected target * @@ -42,6 +55,7 @@ class CNAME extends Validator try { $records = \dns_get_record($domain, DNS_CNAME); + $this->logs = $records; } catch (\Throwable $th) { return false; } diff --git a/src/Appwrite/Platform/Tasks/SSL.php b/src/Appwrite/Platform/Tasks/SSL.php index 6dbf4dcd70..12cb0d6be2 100644 --- a/src/Appwrite/Platform/Tasks/SSL.php +++ b/src/Appwrite/Platform/Tasks/SSL.php @@ -7,6 +7,7 @@ use Appwrite\Event\Certificate; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Document; +use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; class SSL extends Action @@ -21,19 +22,22 @@ class SSL extends Action $this ->desc('Validate server certificates') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) + ->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true) ->inject('queueForCertificates') - ->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates)); + ->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates)); } - public function action(string $domain, Certificate $queueForCertificates): void + public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void { + $skipCheck = \strval($skipCheck) === 'true'; + Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain); $queueForCertificates ->setDomain(new Document([ 'domain' => $domain ])) - ->setSkipRenewCheck(true) + ->setSkipRenewCheck($skipCheck) ->trigger(); } } diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 04ce35daee..75912c32ef 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -23,6 +23,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Domains\Domain; use Utopia\Locale\Locale; +use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -45,7 +46,8 @@ class Certificates extends Action ->inject('queueForMails') ->inject('queueForEvents') ->inject('queueForFunctions') - ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions)); + ->inject('log') + ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); } /** @@ -58,7 +60,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -70,7 +72,7 @@ class Certificates extends Action $domain = new Domain($document->getAttribute('domain', '')); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); } /** @@ -84,7 +86,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -138,11 +140,11 @@ class Certificates extends Action if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain); + $this->validateDomain($domain, $isMainDomain, $log); } // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { + if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) { throw new Exception('Renew isn\'t required.'); } @@ -180,6 +182,8 @@ class Certificates extends Action // Send email to security email $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails); + + throw $e; } finally { // All actions result in new updatedAt date $certificate->setAttribute('updated', DateTime::now()); @@ -247,7 +251,7 @@ class Certificates extends Action * @return void * @throws Exception */ - private function validateDomain(Domain $domain, bool $isMainDomain): void + private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void { if (empty($domain->get())) { throw new Exception('Missing certificate domain.'); @@ -267,8 +271,15 @@ class Certificates extends Action } // Verify domain with DNS records + $validationStart = \microtime(true); $validator = new CNAME($target->get()); if (!$validator->isValid($domain->get())) { + $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); + $log->addTag('dnsDomain', $domain->get()); + + $error = $validator->getLogs(); + $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); + throw new Exception('Failed to verify domain DNS records.'); } } else { @@ -284,7 +295,7 @@ class Certificates extends Action * @return bool True, if certificate needs to be renewed * @throws Exception */ - private function isRenewRequired(string $domain): bool + private function isRenewRequired(string $domain, Log $log): bool { $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; if (\file_exists($certPath)) { @@ -294,12 +305,15 @@ class Certificates extends Action $validTo = $certData['validTo_time_t'] ?? 0; if (empty($validTo)) { + $log->addTag('certificateDomain', $domain); throw new Exception('Unable to read certificate file (cert.pem).'); } // LetsEncrypt allows renewal 30 days before expiry $expiryInAdvance = (60 * 60 * 24 * 30); if ($validTo - $expiryInAdvance > \time()) { + $log->addTag('certificateDomain', $domain); + $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); return false; } }