diff --git a/app/cli.php b/app/cli.php index d2c6496dc3..8902aad886 100644 --- a/app/cli.php +++ b/app/cli.php @@ -5,6 +5,9 @@ require_once __DIR__.'/controllers/general.php'; use Utopia\App; use Utopia\CLI\CLI; use Utopia\CLI\Console; +use Utopia\Database\Validator\Authorization; + +Authorization::disable(); $cli = new CLI(); diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index c00740b130..e533d82795 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -27,6 +27,12 @@ "emails.invitation.footer": "If you are not interested, you can ignore this message.", "emails.invitation.thanks": "Thanks", "emails.invitation.signature": "{{project}} team", + "emails.certificate.subject": "Certificate failure for %s", + "emails.certificate.hello": "Hello", + "emails.certificate.body": "Certificate for your domain '{{domain}}' could not be renewed. This is attempt no. {{attempt}}, and the failure was caused by: {{error}}", + "emails.certificate.footer": "Certificate will still be valid for 30 days since first failure. We highly recommend investigating the case, otherwise your domain will end up without a secure certificate.", + "emails.certificate.thanks": "Thanks", + "emails.certificate.signature": "{{project}} team", "locale.country.unknown": "Unknown", "countries.af": "Afghanistan", "countries.ao": "Angola", diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 38e3fc31c5..b089054b74 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -1331,8 +1331,7 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') $dbForConsole->deleteCachedDocument('projects', $project->getId()); // Issue a TLS certificate when domain is verified - Resque::enqueue('v1-certificates', 'CertificatesV1', [ - 'document' => $domain->getArrayCopy(), + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ 'domain' => $domain->getAttribute('domain'), ]); diff --git a/app/controllers/general.php b/app/controllers/general.php index eacf926955..540c087dae 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -99,16 +99,13 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons ]); $domainDocument = $dbForConsole->createDocument('domains', $domainDocument); - - Console::info('Issuing a TLS certificate for the main domain (' . $domain->get() . ') in a few seconds...'); - - Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ - 'document' => $domainDocument, - 'domain' => $domain->get(), - 'validateTarget' => false, - 'validateCNAME' => false, - ]); } + + Console::info('Issuing a TLS certificate for the main domain (' . $domain->get() . ') in a few seconds...'); + + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ + 'domain' => $domain->get() + ]); } $domains[$domain->get()] = true; diff --git a/app/init.php b/app/init.php index 0c6c366fd7..269d860fe6 100644 --- a/app/init.php +++ b/app/init.php @@ -127,6 +127,7 @@ const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; const MAIL_TYPE_RECOVERY = 'recovery'; const MAIL_TYPE_INVITATION = 'invitation'; +const MAIL_TYPE_CERTIFICATE = 'certificate'; // Auth Types const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a6f37ff71b..056ee59fcd 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -1,10 +1,42 @@ get('cache'))); + $database = new Database(new MariaDB($register->get('db')), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace('_console'); // Main DB + break; // leave loop if successful + } catch(\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: '. $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return $database; +} $cli ->task('maintenance') @@ -54,6 +86,29 @@ $cli ]); } + function renewCertificates($dbForConsole) + { + $time = date('d-m-Y H:i:s', time()); + /** @var Utopia\Database\Database $dbForConsole */ + + $certificates = $dbForConsole->find('certificates', [ + new Query('attempts', Query::TYPE_LESSEREQUAL, [5]), // Maximum 5 attempts + new Query('renewDate', Query::TYPE_LESSEREQUAL, [\time()]) // includes 60 days cooldown (we have 30 days to renew) + ], 200); // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) + + if(\count($certificates) > 0) { + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + foreach ($certificates as $certificate) { + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ + 'domain' => $certificate->getAttribute('domain'), + ]); + } + } else { + Console::info("[{$time}] No certificates for renewal."); + } + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); @@ -62,13 +117,17 @@ $cli $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { + Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { + $database = getConsoleDB(); + $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); + Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); notifyDeleteConnections(); + + renewCertificates($database); }, $interval); }); \ No newline at end of file diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php index 2c32324fa3..f28c0b8532 100644 --- a/app/tasks/ssl.php +++ b/app/tasks/ssl.php @@ -2,22 +2,21 @@ global $cli; +use Appwrite\Event\Event; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Validator\Hostname; $cli ->task('ssl') ->desc('Validate server certificates') - ->action(function () { - $domain = App::getEnv('_APP_DOMAIN', ''); + ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) + ->action(function ($domain) { + Console::success('Scheduling a job to issue a TLS certificate for domain:' . $domain); - Console::log('Issue a TLS certificate for master domain ('.$domain.') in 30 seconds. - Make sure your domain points to your server or restart to try again.'); - - ResqueScheduler::enqueueAt(\time() + 30, 'v1-certificates', 'CertificatesV1', [ - 'document' => [], + // Scheduje a job + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ 'domain' => $domain, - 'validateTarget' => false, - 'validateCNAME' => false, + 'skipRenewCheck' => true ]); }); \ No newline at end of file diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 7fc5b2b727..c6f7726058 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -342,6 +342,7 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -473,6 +474,8 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_DOMAIN_TARGET - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 1df77f9827..f1df74972c 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -1,5 +1,6 @@ getConsoleDB(); - - /** - * 1. Get new domain document - DONE - * 1.1. Validate domain is valid, public suffix is known and CNAME records are verified - DONE - * 2. Check if a certificate already exists - DONE - * 3. Check if certificate is about to expire, if not - skip it - * 3.1. Create / renew certificate - * 3.2. Update loadblancer - * 3.3. Update database (domains, change date, expiry) - * 3.4. Set retry on failure - * 3.5. Schedule to renew certificate in 60 days - */ - Authorization::disable(); - // Args - $document = $this->args['document']; - $domain = $this->args['domain']; + $this->dbForConsole = $this->getConsoleDB(); - // Validation Args - $validateTarget = $this->args['validateTarget'] ?? true; - $validateCNAME = $this->args['validateCNAME'] ?? true; + /** + * 1. Read arguments and validate domain + * 2. Get main domain + * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) + * 4. Validate security email. Cannot be empty, required by LetsEncrypt + * 5. Validate renew date with certificate file, unless requested to skip by parameter + * 6. Issue a certificate using certbot CLI + * 7. Update 'log' attribute on certificate document with Certbot message + * 8. Create storage folder for certificate, if not ready already + * 9. Move certificates from Certbot location to our Storage + * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 11. Read certificate file and update 'renewDate' on certificate document + * 12. Update 'issueDate' and 'attempts' on certificate + * + * If at any point unexpected error occurs, program stops without applying changes to document, and error is thrown into worker + * + * If code stops with expected error: + * 1. 'log' attribute on document is updated with error message + * 2. 'attempts' amount is increased + * 3. Console log is shown + * 4. Email is sent to security email + * + * Unless unexpected error occurs, at the end, we: + * 1. Update 'updated' attribute on document + * 2. Save document to database + * 3. Update all domains documents with current certificate ID + * + * Note: Renewals are checked and scheduled from maintenence worker + */ - // Options - $domain = new Domain((!empty($domain)) ? $domain : ''); - $expiry = 60 * 60 * 24 * 30 * 2; // 60 days - $safety = 60 * 60; // 1 hour - $renew = (\time() + $expiry); + try { + // Read arguments + $domain = $this->args['domain']; // String of domain (hostname) + $skipRenewCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file - if (empty($domain->get())) { - throw new Exception('Missing domain'); - } + $domain = new Domain((!empty($domain)) ? $domain : ''); - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain'); - } - - if ($validateTarget) { - $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); - - if(!$target->isKnown() || $target->isTest()) { - throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); - } - } - - if ($validateCNAME) { - $validator = new CNAME($target->get()); // Verify Domain with DNS records - - if(!$validator->isValid($domain->get())) { - throw new Exception('Failed to verify domain DNS records'); - } - } - - $certificate = $dbForConsole->findOne('certificates', [ - new Query('domain', QUERY::TYPE_EQUAL, [$domain->get()]) - ]); - - // $condition = ($certificate - // && $certificate instanceof Document - // && isset($certificate['issueDate']) - // && (($certificate['issueDate'] + ($expiry)) > time())) ? 'true' : 'false'; - - // throw new Exception('cert issued at'.date('d.m.Y H:i', $certificate['issueDate']).' | renew date is: '.date('d.m.Y H:i', ($certificate['issueDate'] + ($expiry))).' | condition is '.$condition); - - $certificate = (!empty($certificate) && $certificate instanceof $certificate) ? $certificate->getArrayCopy() : []; - - if ( - !empty($certificate) - && isset($certificate['issueDate']) - && (($certificate['issueDate'] + ($expiry)) > \time()) - ) { // Check last issue time - - // Update document anyway, if needed. - // This occurs when a cert is already generated because a different project is using the domain. - // By updating here we ensure all domains has certificateId assigned (share same certificate document) - if(!isset($document['certificateId'])) { - $certificate = new Document($certificate); - - $domain = new Document(\array_merge($document, [ - 'updated' => \time(), - 'certificateId' => $certificate->getId(), - ])); - - $domain = $dbForConsole->updateDocument('domains', $domain->getId(), $domain); - - if(!$certificate) { - throw new Exception('Failed saving domain to DB'); - } - } - - throw new Exception('Renew isn\'t required'); - } - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); - - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate'); - } - - $stdout = ''; - $stderr = ''; - - $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain->get()}", '', $stdout, $stderr); - - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - $path = APP_STORAGE_CERTIFICATES . '/' . $domain->get(); - - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path...'); - } - } - - if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem: '.\json_encode($stdout)); - } - - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem: ' . \json_encode($stdout)); - } - - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout)); - } - - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem: ' . \json_encode($stdout)); - } - - $certificate = new Document(\array_merge($certificate, [ - 'domain' => $domain->get(), - 'issueDate' => \time(), - 'renewDate' => $renew, - 'attempts' => 0, - 'log' => \json_encode($stdout), - ])); - - $certificate = $dbForConsole->createDocument('certificates', $certificate); - - if (!$certificate) { - throw new Exception('Failed saving certificate to DB'); - } - - if(!empty($document)) { - $certificate = new Document(\array_merge($document, [ - 'updated' => \time(), - 'certificateId' => $certificate->getId(), - ])); - - $certificate = $dbForConsole->updateDocument('domains', $certificate->getId(), $certificate); + // Get current certificate + $certificate = $this->dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) ]); + // If we don't have certificate for domain yet, let's create new document. At the end we save it if(!$certificate) { - throw new Exception('Failed saving domain to DB'); + $certificate = new Document(); + $certificate->setAttribute('domain', $domain->get()); } + + // Email for alerts is required by LetsEncrypt + $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); + } + + $mainDomain = $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + $this->validateDomain($domain, $isMainDomain); + + // If certificate exists already, double-check expiry date + // If asked to skip, we won't + if(!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { + throw new Exception('Renew isn\'t required.'); + } + + // Generate certificate files using Let's Encrypt + $letsEncryptData = $this->issueCertificate($domain->get(), $email); + + // Command succeeded, store all data into document + // We store stderr too, because it may include warnings + $certificate->setAttribute('log', \json_encode([ + 'stdout' => $letsEncryptData['stdout'], + 'stderr' => $letsEncryptData['stderr'], + ])); + + // Give certificates to Traefik + $this->applyCertificateFiles($domain->get()); + + // Update certificate info stored in database + $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('attempts', 0); + $certificate->setAttribute('issueDate', \time()); + } catch(Throwable $e) { + // Set exception as log in certificate document + $certificate->setAttribute('log', $e->getMessage()); + + // Increase attempts count + $attempts = $certificate->getAttribute('attempts', 0) + 1; + $certificate->setAttribute('attempts', $attempts); + + // Send email to security email + $this->notifyError($domain->get(), $e->getMessage(), $attempts); + } finally { + // All actions result in new updatedAt date + $certificate->setAttribute('updated', \time()); + + // Save all changes we made to certificate document into database + $this->saveCertificateDocument($domain->get(), $certificate); + + Authorization::reset(); } - - $config = -"tls: - certificates: - - certFile: /storage/certificates/{$domain->get()}/fullchain.pem - keyFile: /storage/certificates/{$domain->get()}/privkey.pem"; - - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain->get() . '.yml', $config)) { - throw new Exception('Failed to save SSL configuration'); - } - - ResqueScheduler::enqueueAt($renew + $safety, 'v1-certificates', 'CertificatesV1', [ - 'document' => [], - 'domain' => $domain->get(), - 'validateTarget' => $validateTarget, - 'validateCNAME' => $validateCNAME, - ]); // Async task rescheduale - - Authorization::reset(); } public function shutdown(): void { } + + /** + * Save certificate data into database. + * + * @param string $domain Domain name that certificate is for + * @param Document $certificate Certificate document that we need to save + * + * @return void + */ + private function saveCertificateDocument(string $domain, Document $certificate): void { + // Check if update or insert required + $certificateDocument = $this->dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain]) ]); + if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { + // Merge new data with current data + $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); + + $certificate = $this->dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); + } else { + $certificate = $this->dbForConsole->createDocument('certificates', $certificate); + } + + $certificateId = $certificate->getId(); + $this->updateDomainDocuments($certificateId, $domain); + } + + /** + * Get main domain. Needed as we do different checks for main and non-main domains. + * + * @return null|string Returns main domain. If null, there is no main domain yet. + */ + private function getMainDomain(): ?string { + if (!empty(App::getEnv('_APP_DOMAIN', ''))) { + return App::getEnv('_APP_DOMAIN', ''); + } else { + $domainDocument = $this->dbForConsole->findOne('domains', [], 0, ['_id'], ['ASC']); + if($domainDocument) { + return $domainDocument->getAttribute('domain'); + } + } + + return null; + } + + /** + * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * - Domain must have proper DNS record + * + * @param Domain $domain Domain which we validate + * @param bool $isMainDomain In case of master domain, we look for different DNS configurations + * + * @return void + */ + private function validateDomain(Domain $domain, bool $isMainDomain): void { + if (empty($domain->get())) { + throw new Exception('Missing certificate domain.'); + } + + if (!$domain->isKnown() || $domain->isTest()) { + throw new Exception('Unknown public suffix for domain.'); + } + + if (!$isMainDomain) { + // TODO: Would be awesome to also support A/AAAA records here. Maybe dry run? + + // Validate if domain target is properly configured + $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); + + if (!$target->isKnown() || $target->isTest()) { + throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); + } + + // Verify domain with DNS records + $validator = new CNAME($target->get()); + if (!$validator->isValid($domain->get())) { + throw new Exception('Failed to verify domain DNS records.'); + } + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + } + } + + /** + * Reads expiry date of certificate from file and decides if renewal is required or not. + * + * @param string $domain Domain for which we check certificate file + * + * @return bool True, if certificate needs to be renewed + */ + private function isRenewRequired(string $domain): bool { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $validTo = null; + + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60*60*24*30); + if ($validTo - $expiryInAdvance > \time()) { + return false; + } + } + + return true; + } + + /** + * LetsEncrypt communication to issue certificate (using certbot CLI) + * + * @param string $domain Domain to generate certificate for + * + * @return array Named array with keys 'stdout' and 'stderr', both string + */ + private function issueCertificate(string $domain, string $email): array { + $staging = (App::isProduction()) ? '' : ' --dry-run'; + + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $email + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", '', $stdout, $stderr); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + return [ + 'stdout' => $stdout, + 'stderr' => $stderr + ]; + } + + /** + * Read new renew date from certificate file generated by Let's Encrypt + * + * @param string $domain Domain which certificate was generated for + * + * @return int + */ + private function getRenewDate(string $domain): int { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + $expiryInAdvance = (60*60*24*30); // 30 days + return $validTo - $expiryInAdvance; + } + + /** + * Method to take files from Let's Encrypt, and put it into Traefik. + * + * @param string $domain Domain which certificate was generated for + * + * @return void + */ + private function applyCertificateFiles(string $domain): void { + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files from certbot into our storage + if(!@\rename('/etc/letsencrypt/live/'.$domain.'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain.'/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem.'); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem.'); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem.'); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem.'); + } + + $config = + "tls:" . + " certificates:" . + " - certFile: /storage/certificates/{$domain}/fullchain.pem" . + " keyFile: /storage/certificates/{$domain}/privkey.pem"; + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + } + + /** + * Method to make sure information about error is delivered to admnistrator. + * + * @param string $domain Domain that caused the error + * @param string $errorMessage Verbose error message + * @param int $attempt How many times it failed already + * + * @return void + */ + private function notifyError(string $domain, string $errorMessage, int $attempt): void { + // Log error into console + Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage); + + // Send mail to administratore mail + Resque::enqueue(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME, [ + 'from' => 'console', + 'project' => 'console', + 'name' => 'Appwrite Administrator', + 'recipient' => App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'), + 'url' => 'https://' . $domain, + 'locale' => App::getEnv('_APP_LOCALE', 'en'), + 'type' => MAIL_TYPE_CERTIFICATE, + + 'domain' => $domain, + 'error' => $errorMessage, + 'attempt' => $attempt + ]); + } + + /** + * Update all existing domain documents so they have relation to correct certificate document. + * This solved issues: + * - when adding a domain for which there is already a certificate + * - when renew creates new document? It might? + * - overall makes it more reliable + * + * @param string $certificateId ID of a new or updated certificate document + * @param string $domain Domain that is affected by new certificate + * + * @return void + */ + private function updateDomainDocuments(string $certificateId, string $domain): void { + $domains = $this->dbForConsole->find('domains', [ + new Query('domain', Query::TYPE_EQUAL, [$domain]) + ], 1000); + + foreach ($domains as $domainDocument) { + $domainDocument->setAttribute('updated', \time()); + $domainDocument->setAttribute('certificateId', $certificateId); + + $this->dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument); + $this->dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId')); + } + } } diff --git a/app/workers/mails.php b/app/workers/mails.php index 25ddb438c0..6905593a8c 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -46,6 +46,16 @@ class MailsV1 extends Worker $body = Template::fromFile(__DIR__ . '/../config/locale/templates/email-base.tpl'); $subject = ''; switch ($type) { + case MAIL_TYPE_CERTIFICATE: + $domain = $this->args['domain']; + $error = $this->args['error']; + $attempt = $this->args['attempt']; + + $subject = \sprintf($locale->getText("$prefix.subject"), $domain); + $body->setParam('{{domain}}', $domain); + $body->setParam('{{error}}', $error); + $body->setParam('{{attempt}}', $attempt); + break; case MAIL_TYPE_INVITATION: $subject = \sprintf($locale->getText("$prefix.subject"), $this->args['team'], $project); $body->setParam('{{owner}}', $this->args['owner']); @@ -126,6 +136,8 @@ class MailsV1 extends Worker switch ($type) { case MAIL_TYPE_RECOVERY: return 'emails.recovery'; + case MAIL_TYPE_CERTIFICATE: + return 'emails.certificate'; case MAIL_TYPE_INVITATION: return 'emails.invitation'; case MAIL_TYPE_VERIFICATION: diff --git a/composer.lock b/composer.lock index ea07e51fea..c2b51c52d9 100644 --- a/composer.lock +++ b/composer.lock @@ -2299,25 +2299,25 @@ }, { "name": "utopia-php/image", - "version": "0.5.3", + "version": "0.5.4", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "4a8429b62dcf56562b038d6712375f75166f0c02" + "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/4a8429b62dcf56562b038d6712375f75166f0c02", - "reference": "4a8429b62dcf56562b038d6712375f75166f0c02", + "url": "https://api.github.com/repos/utopia-php/image/zipball/ca5f436f9aa22dedaa6648f24f3687733808e336", + "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336", "shasum": "" }, "require": { "ext-imagick": "*", - "php": ">=7.4" + "php": ">=8.0" }, "require-dev": { "phpunit/phpunit": "^9.3", - "vimeo/psalm": "4.0.1" + "vimeo/psalm": "4.13.1" }, "type": "library", "autoload": { @@ -2345,9 +2345,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.5.3" + "source": "https://github.com/utopia-php/image/tree/0.5.4" }, - "time": "2021-11-02T05:47:16+00:00" + "time": "2022-05-11T12:30:41+00:00" }, { "name": "utopia-php/locale", @@ -3551,16 +3551,16 @@ }, { "name": "matthiasmullie/minify", - "version": "1.3.67", + "version": "1.3.68", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9" + "reference": "c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", - "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297", + "reference": "c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297", "shasum": "" }, "require": { @@ -3609,7 +3609,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.67" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.68" }, "funding": [ { @@ -3617,7 +3617,7 @@ "type": "github" } ], - "time": "2022-03-24T08:54:59+00:00" + "time": "2022-04-19T08:28:56+00:00" }, { "name": "matthiasmullie/path-converter", @@ -5711,16 +5711,16 @@ }, { "name": "symfony/console", - "version": "v6.0.7", + "version": "v6.0.8", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e" + "reference": "0d00aa289215353aa8746a31d101f8e60826285c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e", - "reference": "70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e", + "url": "https://api.github.com/repos/symfony/console/zipball/0d00aa289215353aa8746a31d101f8e60826285c", + "reference": "0d00aa289215353aa8746a31d101f8e60826285c", "shasum": "" }, "require": { @@ -5786,7 +5786,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.0.7" + "source": "https://github.com/symfony/console/tree/v6.0.8" }, "funding": [ { @@ -5802,7 +5802,7 @@ "type": "tidelift" } ], - "time": "2022-03-31T17:18:25+00:00" + "time": "2022-04-20T15:01:42+00:00" }, { "name": "symfony/polyfill-intl-grapheme", @@ -6136,16 +6136,16 @@ }, { "name": "symfony/string", - "version": "v6.0.3", + "version": "v6.0.8", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "522144f0c4c004c80d56fa47e40e17028e2eefc2" + "reference": "ac0aa5c2282e0de624c175b68d13f2c8f2e2649d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/522144f0c4c004c80d56fa47e40e17028e2eefc2", - "reference": "522144f0c4c004c80d56fa47e40e17028e2eefc2", + "url": "https://api.github.com/repos/symfony/string/zipball/ac0aa5c2282e0de624c175b68d13f2c8f2e2649d", + "reference": "ac0aa5c2282e0de624c175b68d13f2c8f2e2649d", "shasum": "" }, "require": { @@ -6201,7 +6201,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.0.3" + "source": "https://github.com/symfony/string/tree/v6.0.8" }, "funding": [ { @@ -6217,7 +6217,7 @@ "type": "tidelift" } ], - "time": "2022-01-02T09:55:41+00:00" + "time": "2022-04-22T08:18:02+00:00" }, { "name": "textalk/websocket", diff --git a/docker-compose.yml b/docker-compose.yml index 0667f5afc5..1a00132914 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,6 +389,7 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -543,6 +544,11 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_ABUSE