From 752079c87455cd895076cb15c65f5ea607b8ae4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 10 Apr 2022 09:38:22 +0000 Subject: [PATCH 01/10] Refactored SSL genration --- app/cli.php | 3 + app/controllers/api/projects.php | 4 +- app/controllers/general.php | 8 +- app/tasks/maintenance.php | 96 ++++++- app/tasks/ssl.php | 12 +- app/views/install/compose.phtml | 3 + app/workers/certificates.php | 351 ++++++++++++++----------- composer.lock | 115 ++++---- docker-compose.yml | 6 + src/Appwrite/Exception/Certificate.php | 38 +++ 10 files changed, 412 insertions(+), 224 deletions(-) create mode 100644 src/Appwrite/Exception/Certificate.php diff --git a/app/cli.php b/app/cli.php index d2c6496dc3..8902aad886 100644 --- a/app/cli.php +++ b/app/cli.php @@ -5,6 +5,9 @@ require_once __DIR__.'/controllers/general.php'; use Utopia\App; use Utopia\CLI\CLI; use Utopia\CLI\Console; +use Utopia\Database\Validator\Authorization; + +Authorization::disable(); $cli = new CLI(); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 2bee190628..eefabe43a1 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -2,6 +2,7 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; +use Appwrite\Event\Event; use Appwrite\Network\Validator\CNAME; use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\Origin; @@ -1389,8 +1390,7 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') $dbForConsole->deleteCachedDocument('projects', $project->getId()); // Issue a TLS certificate when domain is verified - Resque::enqueue('v1-certificates', 'CertificatesV1', [ - 'document' => $domain->getArrayCopy(), + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ 'domain' => $domain->getAttribute('domain'), ]); diff --git a/app/controllers/general.php b/app/controllers/general.php index a65e07c81c..3092e537c1 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,6 +12,7 @@ use Appwrite\Extend\Exception; use Utopia\Config\Config; use Utopia\Domains\Domain; use Appwrite\Auth\Auth; +use Appwrite\Event\Event; use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Response\Filters\V11 as ResponseV11; use Appwrite\Utopia\Response\Filters\V12 as ResponseV12; @@ -90,11 +91,8 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons Console::info('Issuing a TLS certificate for the master domain (' . $domain->get() . ') in a few seconds...'); - Resque::enqueue('v1-certificates', 'CertificatesV1', [ - 'document' => $domainDocument, - 'domain' => $domain->get(), - 'validateTarget' => false, - 'validateCNAME' => false, + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ + 'domain' => $domain->get() ]); } diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a6f37ff71b..a50d8c4da9 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -1,15 +1,58 @@ get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + + $cache = new Cache(new RedisCache($redis)); + $database = new Database(new MariaDB($db), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace('_console'); + + break; // leave loop if successful + } catch(\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: '. $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return [ + $database, + function () use ($register, $db, $redis) { + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); + } + ]; + +}; $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') - ->action(function () { + ->action(function () use ($register) { Console::title('Maintenance V1'); Console::success(APP_NAME.' maintenance process v1 has started'); @@ -54,6 +97,29 @@ $cli ]); } + function renewCertificates($dbForConsole) + { + $time = date('d-m-Y H:i:s', time()); + /** @var Utopia\Database\Database $dbForConsole */ + + $certificates = $dbForConsole->find('certificates', [ + new Query('attempts', Query::TYPE_LESSER, [5]), // Maximum 5 attempts + new Query('renewDate', Query::TYPE_LESSEREQUAL, [\time()]) // includes 60 days cooldown (we have 30 days to renew) + ], 300); // Limit 300 comes from LetsEncrypt (orders per 3 hours) + + if(\count($certificates) > 0) { + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + foreach ($certificates as $certificate) { + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ + 'domain' => $certificate->getAttribute('domain'), + ]); + } + } else { + Console::info("[{$time}] No certificates for renewal."); + } + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); @@ -62,13 +128,25 @@ $cli $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); - notifyDeleteExecutionLogs($executionLogsRetention); - notifyDeleteAbuseLogs($abuseLogsRetention); - notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); - notifyDeleteConnections(); + Console::loop(function() use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { + go(function () use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { + try { + [$database, $returnDatabase] = getDatabase($register, '_console'); + + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); + notifyDeleteExecutionLogs($executionLogsRetention); + notifyDeleteAbuseLogs($abuseLogsRetention); + notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); + notifyDeleteConnections(); + + renewCertificates($database); + } catch (\Throwable $th) { + throw $th; + } finally { + call_user_func($returnDatabase); + } + }); }, $interval); }); \ No newline at end of file diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php index 2c32324fa3..6010c3fd1f 100644 --- a/app/tasks/ssl.php +++ b/app/tasks/ssl.php @@ -11,13 +11,15 @@ $cli ->action(function () { $domain = App::getEnv('_APP_DOMAIN', ''); - Console::log('Issue a TLS certificate for master domain ('.$domain.') in 30 seconds. + // TODO: Instead of waiting, let's ping Traefik. If responds, we can schedule instantly + // TODO: Add support for argument (domain) + + Console::log('Issue a TLS certificate for master domain ('.$domain.') in 2 seconds. Make sure your domain points to your server or restart to try again.'); - ResqueScheduler::enqueueAt(\time() + 30, 'v1-certificates', 'CertificatesV1', [ - 'document' => [], + // Const for types not available here + ResqueScheduler::enqueueAt(\time() + 2, 'v1-certificates', 'CertificatesV1', [ 'domain' => $domain, - 'validateTarget' => false, - 'validateCNAME' => false, + 'skipRenewCheck' => true // TODO: Discuss this behabiour. true? false? parameter? How do we document it? ]); }); \ No newline at end of file diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 5740c8680d..bb3fa8d8f3 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -449,6 +449,9 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_DOMAIN_TARGET + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 9f9ce33dbd..fea025d163 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -1,7 +1,9 @@ getConsoleDB(); - - /** - * 1. Get new domain document - DONE - * 1.1. Validate domain is valid, public suffix is known and CNAME records are verified - DONE - * 2. Check if a certificate already exists - DONE - * 3. Check if certificate is about to expire, if not - skip it - * 3.1. Create / renew certificate - * 3.2. Update loadblancer - * 3.3. Update database (domains, change date, expiry) - * 3.4. Set retry on failure - * 3.5. Schedule to renew certificate in 60 days - */ - Authorization::disable(); - // Args - $document = $this->args['document']; - $domain = $this->args['domain']; + $dbForConsole = $this->getConsoleDB(); - // Validation Args - $validateTarget = $this->args['validateTarget'] ?? true; - $validateCNAME = $this->args['validateCNAME'] ?? true; + $certificate = new Document(); - // Options - $domain = new Domain((!empty($domain)) ? $domain : ''); - $expiry = 60 * 60 * 24 * 30 * 2; // 60 days - $safety = 60 * 60; // 1 hour - $renew = (\time() + $expiry); + try { + /** + * TODO: Update + * 1. Get new domain document - DONE + * 1.1. Validate domain is valid, public suffix is known and CNAME records are verified - DONE + * 2. Check if a certificate already exists - DONE + * 3. Check if certificate is about to expire, if not - skip it + * 3.1. Create / renew certificate + * 3.2. Update loadblancer + * 3.3. Update database (domains, change date, expiry) + * 3.4. Set retry on failure + * 3.5. Schedule to renew certificate in 60 days + */ + + + // Get attributes + $domain = $this->args['domain']; // String of domain (hostname) + $domain = new Domain((!empty($domain)) ? $domain : ''); - if (empty($domain->get())) { - throw new Exception('Missing domain'); - } - - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain'); - } - - if ($validateTarget) { - $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); - - if(!$target->isKnown() || $target->isTest()) { - throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); + $certificate->setAttribute('domain', $domain->get()); + + $skipRenewCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file + + $mainDomain = null; // ENV or first ever visited domain + if (!empty(App::getEnv('_APP_DOMAIN', ''))) { + $mainDomain = App::getEnv('_APP_DOMAIN', ''); + } else { + $domainDocument = $dbForConsole->findOne('domains', [], 0, ['_id'], ['ASC']); + $mainDomain = $domainDocument ? $domainDocument->getAttribute('domain') : $domain->get(); } - } - - if ($validateCNAME) { - $validator = new CNAME($target->get()); // Verify Domain with DNS records - - if(!$validator->isValid($domain->get())) { - throw new Exception('Failed to verify domain DNS records'); + + // If not main domain, we will check CNAME record + $validateCNAME = false; + if ($domain->get() !== $mainDomain) { + $validateCNAME = true; } - } - - $certificate = $dbForConsole->findOne('certificates', [ - new Query('domain', QUERY::TYPE_EQUAL, [$domain->get()]) - ]); - - // $condition = ($certificate - // && $certificate instanceof Document - // && isset($certificate['issueDate']) - // && (($certificate['issueDate'] + ($expiry)) > time())) ? 'true' : 'false'; - - // throw new Exception('cert issued at'.date('d.m.Y H:i', $certificate['issueDate']).' | renew date is: '.date('d.m.Y H:i', ($certificate['issueDate'] + ($expiry))).' | condition is '.$condition); - - $certificate = (!empty($certificate) && $certificate instanceof $certificate) ? $certificate->getArrayCopy() : []; - - if ( - !empty($certificate) - && isset($certificate['issueDate']) - && (($certificate['issueDate'] + ($expiry)) > \time()) - ) { // Check last issue time - throw new Exception('Renew isn\'t required'); - } - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); - - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate'); - } - - $stdout = ''; - $stderr = ''; - - $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain->get()}", '', $stdout, $stderr); - - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - $path = APP_STORAGE_CERTIFICATES . '/' . $domain->get(); - - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path...'); + + if (empty($domain->get())) { + throw new ExceptionCertificate('Missing certificate domain.'); + } + + if (!$domain->isKnown() || $domain->isTest()) { + throw new ExceptionCertificate('Unknown public suffix for domain.'); + } + + if ($validateCNAME) { + // TODO: Would be awesome to also support A/AAAA records here. Maybe dry run? + + // Validate if domain target is properly configured + $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); + + if (!$target->isKnown() || $target->isTest()) { + throw new ExceptionCertificate('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); + } + + // Verify domain with DNS records + $validator = new CNAME($target->get()); + if (!$validator->isValid($domain->get())) { + throw new ExceptionCertificate('Failed to verify domain DNS records.'); + } + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? } - } - if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem: '.\json_encode($stdout)); - } + // If certificate exists already, double-check expiry date + // If asked to skip, we won't + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/cert.pem'; + if (!$skipRenewCheck && \file_exists($certPath)) { + $validTo = null; - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem: ' . \json_encode($stdout)); - } + try { + $certData = openssl_x509_parse(file_get_contents($certPath)); + + $validTo = $certData['validTo_time_t']; + + if (empty($validTo)) { + throw new Exception('Invalid expiry date.'); + } + } catch(\Throwable $th) { + throw new ExceptionCertificate('Unable to read certificate file (cert.pem).'); + } - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout)); - } + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60*60*24*30); + if ($validTo - $expiryInAdvance > \time()) { + $validToVerbose = date('d-m-Y H:i:s', $validTo); + throw new ExceptionCertificate('Renew isn\'t required. Next renew at ' . $validToVerbose); + } + } + + // Email for alerts is required by LetsEncrypt + $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); + if (empty($email)) { + throw new ExceptionCertificate('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); + } - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem: ' . \json_encode($stdout)); - } + // LetsEncrypt communication to issue certificate (using certbot CLI) + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $email + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain->get()}", '', $stdout, $stderr); + + // All exceptions from now on will be marked to increment attempts count. This allows us to only limit attempts for domains that failed on LectEncrypt side. + // Such attempts count allows us to prevent API limit abuse with always failing domains + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new ExceptionCertificate('Failed to issue a certificate with message: ' . $stderr, true); + } - $certificate = new Document(\array_merge($certificate, [ - 'domain' => $domain->get(), - 'issueDate' => \time(), - 'renewDate' => $renew, - 'attempts' => 0, - 'log' => \json_encode($stdout), - ])); - - $certificate = $dbForConsole->createDocument('certificates', $certificate); - - if (!$certificate) { - throw new Exception('Failed saving certificate to DB'); - } - - if(!empty($document)) { - $certificate = new Document(\array_merge($document, [ - 'updated' => \time(), - 'certificateId' => $certificate->getId(), + // Command succeeded, store all data into document + // We store stderr too, because it may include warnings + // This is only stored if everytng below passes too. Otherwise, it will be overwritten by error message + $certificate->setAttribute('log', \json_encode([ + 'stdout' => $stdout, + 'stderr' => $stderr, ])); - - $certificate = $dbForConsole->updateDocument('domains', $certificate->getId(), $certificate); - - if(!$certificate) { - throw new Exception('Failed saving domain to DB'); + + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain->get(); + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new ExceptionCertificate('Failed to create path for certificate.', true); + } } + + // Move generated files from certbot into our storage + if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { + throw new ExceptionCertificate('Failed to rename certificate cert.pem: '.\json_encode($stdout), true); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/chain.pem')) { + throw new ExceptionCertificate('Failed to rename certificate chain.pem: ' . \json_encode($stdout), true); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/fullchain.pem')) { + throw new ExceptionCertificate('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout), true); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/privkey.pem')) { + throw new ExceptionCertificate('Failed to rename certificate privkey.pem: ' . \json_encode($stdout), true); + } + + // This multi-line syntax helps IDE + $config = + "tls:" . + " certificates:" . + " - certFile: /storage/certificates/{$domain->get()}/fullchain.pem" . + " keyFile: /storage/certificates/{$domain->get()}/privkey.pem"; + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain->get() . '.yml', $config)) { + throw new ExceptionCertificate('Failed to save Traefik configuration.', true); + } + + // Read new renew date from cert file + // TODO: This might not be required, we could calculate it. But this feels safer + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t']; + $expiryInAdvance = (60*60*24*30); + $certificate->setAttribute('renewDate', $validTo - $expiryInAdvance); + + // All went well at this point 🥳 + + // Reset attempts count for next renwal + $certificate->setAttribute('attempts', 0); + + // Mark issue date + $certificate->setAttribute('issueDate', \time()); + } catch(ExceptionCertificate $e) { + // These exceptions are expected if renew shouldn't or can't happen + + // Add exception as log into certificate + $certificate->setAttribute('log', $e->getMessage()); + + $attempt = $certificate->getAttribute('attempts', 0); + $attempt++; + + // Save increased attempts count if requested by exception + if($e->getIncrementAttempts()) { + $certificate->setAttribute('attempts', $attempt); + } + + Console::warning('Cannot renew domain (' . $domain->get() . ') on attempt no. ' . $attempt . ' certificate: ' . $e->getMessage()); + } finally { + // All actions result in new updatedAt date + $certificate->setAttribute('updated', \time()); + + // Save certificate data into database + // Check if update or insert required + $certificateDocument = $dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) ]); + if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { + // Merge new data with current data + $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); + + $certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); + } else { + $certificate = $dbForConsole->createDocument('certificates', $certificate); + } + + // Update domains with new certificate ID + $certificateId = $certificate->getId(); + // TODO: Add logic for updating domains + + Authorization::reset(); } - - $config = -"tls: - certificates: - - certFile: /storage/certificates/{$domain->get()}/fullchain.pem - keyFile: /storage/certificates/{$domain->get()}/privkey.pem"; - - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain->get() . '.yml', $config)) { - throw new Exception('Failed to save SSL configuration'); - } - - ResqueScheduler::enqueueAt($renew + $safety, 'v1-certificates', 'CertificatesV1', [ - 'document' => [], - 'domain' => $domain->get(), - 'validateTarget' => $validateTarget, - 'validateCNAME' => $validateCNAME, - ]); // Async task rescheduale - - Authorization::reset(); } public function shutdown(): void diff --git a/composer.lock b/composer.lock index 4a9b786b33..00ea8b8806 100644 --- a/composer.lock +++ b/composer.lock @@ -300,20 +300,23 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.12.1", + "version": "v1.13.0", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "c27faa11724229986335c23f4b6d0f1d8d6547fb" + "reference": "afec8e58ec93d2291c127fa19709a048f28641e5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/c27faa11724229986335c23f4b6d0f1d8d6547fb", - "reference": "c27faa11724229986335c23f4b6d0f1d8d6547fb", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/afec8e58ec93d2291c127fa19709a048f28641e5", + "reference": "afec8e58ec93d2291c127fa19709a048f28641e5", "shasum": "" }, "require": { - "php": ">=5.4.0" + "php": ">=5.6.0" + }, + "suggest": { + "ext-redis": "Improved performance for communicating with redis" }, "type": "library", "autoload": { @@ -338,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.12.1" + "source": "https://github.com/colinmollenhour/credis/tree/v1.13.0" }, - "time": "2020-11-06T16:09:14+00:00" + "time": "2022-04-07T14:57:22+00:00" }, { "name": "composer/package-versions-deprecated", @@ -1580,16 +1583,16 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v3.0.0", + "version": "v3.0.1", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "c726b64c1ccfe2896cb7df2e1331c357ad1c8ced" + "reference": "26954b3d62a6c5fd0ea8a2a00c0353a14978d05c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/c726b64c1ccfe2896cb7df2e1331c357ad1c8ced", - "reference": "c726b64c1ccfe2896cb7df2e1331c357ad1c8ced", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/26954b3d62a6c5fd0ea8a2a00c0353a14978d05c", + "reference": "26954b3d62a6c5fd0ea8a2a00c0353a14978d05c", "shasum": "" }, "require": { @@ -1627,7 +1630,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.0.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.0.1" }, "funding": [ { @@ -1643,7 +1646,7 @@ "type": "tidelift" } ], - "time": "2021-11-01T23:48:49+00:00" + "time": "2022-01-02T09:55:41+00:00" }, { "name": "symfony/polyfill-ctype", @@ -3192,16 +3195,16 @@ }, { "name": "composer/semver", - "version": "3.3.1", + "version": "3.3.2", "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "5d8e574bb0e69188786b8ef77d43341222a41a71" + "reference": "3953f23262f2bff1919fc82183ad9acb13ff62c9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/5d8e574bb0e69188786b8ef77d43341222a41a71", - "reference": "5d8e574bb0e69188786b8ef77d43341222a41a71", + "url": "https://api.github.com/repos/composer/semver/zipball/3953f23262f2bff1919fc82183ad9acb13ff62c9", + "reference": "3953f23262f2bff1919fc82183ad9acb13ff62c9", "shasum": "" }, "require": { @@ -3253,7 +3256,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.3.1" + "source": "https://github.com/composer/semver/tree/3.3.2" }, "funding": [ { @@ -3269,7 +3272,7 @@ "type": "tidelift" } ], - "time": "2022-03-16T11:22:07+00:00" + "time": "2022-04-01T19:23:25+00:00" }, { "name": "composer/xdebug-handler", @@ -3491,16 +3494,16 @@ }, { "name": "felixfbecker/language-server-protocol", - "version": "1.5.1", + "version": "v1.5.2", "source": { "type": "git", "url": "https://github.com/felixfbecker/php-language-server-protocol.git", - "reference": "9d846d1f5cf101deee7a61c8ba7caa0a975cd730" + "reference": "6e82196ffd7c62f7794d778ca52b69feec9f2842" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/felixfbecker/php-language-server-protocol/zipball/9d846d1f5cf101deee7a61c8ba7caa0a975cd730", - "reference": "9d846d1f5cf101deee7a61c8ba7caa0a975cd730", + "url": "https://api.github.com/repos/felixfbecker/php-language-server-protocol/zipball/6e82196ffd7c62f7794d778ca52b69feec9f2842", + "reference": "6e82196ffd7c62f7794d778ca52b69feec9f2842", "shasum": "" }, "require": { @@ -3541,9 +3544,9 @@ ], "support": { "issues": "https://github.com/felixfbecker/php-language-server-protocol/issues", - "source": "https://github.com/felixfbecker/php-language-server-protocol/tree/1.5.1" + "source": "https://github.com/felixfbecker/php-language-server-protocol/tree/v1.5.2" }, - "time": "2021-02-22T14:02:09+00:00" + "time": "2022-03-02T22:36:06+00:00" }, { "name": "matthiasmullie/minify", @@ -4118,16 +4121,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.6.0", + "version": "1.6.1", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "93ebd0014cab80c4ea9f5e297ea48672f1b87706" + "reference": "77a32518733312af16a44300404e945338981de3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/93ebd0014cab80c4ea9f5e297ea48672f1b87706", - "reference": "93ebd0014cab80c4ea9f5e297ea48672f1b87706", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/77a32518733312af16a44300404e945338981de3", + "reference": "77a32518733312af16a44300404e945338981de3", "shasum": "" }, "require": { @@ -4162,9 +4165,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.1" }, - "time": "2022-01-04T19:58:01+00:00" + "time": "2022-03-15T21:29:03+00:00" }, { "name": "phpspec/prophecy", @@ -5073,16 +5076,16 @@ }, { "name": "sebastian/environment", - "version": "5.1.3", + "version": "5.1.4", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "388b6ced16caa751030f6a69e588299fa09200ac" + "reference": "1b5dff7bb151a4db11d49d90e5408e4e938270f7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/388b6ced16caa751030f6a69e588299fa09200ac", - "reference": "388b6ced16caa751030f6a69e588299fa09200ac", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/1b5dff7bb151a4db11d49d90e5408e4e938270f7", + "reference": "1b5dff7bb151a4db11d49d90e5408e4e938270f7", "shasum": "" }, "require": { @@ -5124,7 +5127,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/environment/issues", - "source": "https://github.com/sebastianbergmann/environment/tree/5.1.3" + "source": "https://github.com/sebastianbergmann/environment/tree/5.1.4" }, "funding": [ { @@ -5132,7 +5135,7 @@ "type": "github" } ], - "time": "2020-09-28T05:52:38+00:00" + "time": "2022-04-03T09:37:03+00:00" }, { "name": "sebastian/exporter", @@ -5715,16 +5718,16 @@ }, { "name": "symfony/console", - "version": "v6.0.5", + "version": "v6.0.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3bebf4108b9e07492a2a4057d207aa5a77d146b1" + "reference": "70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3bebf4108b9e07492a2a4057d207aa5a77d146b1", - "reference": "3bebf4108b9e07492a2a4057d207aa5a77d146b1", + "url": "https://api.github.com/repos/symfony/console/zipball/70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e", + "reference": "70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e", "shasum": "" }, "require": { @@ -5790,7 +5793,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.0.5" + "source": "https://github.com/symfony/console/tree/v6.0.7" }, "funding": [ { @@ -5806,7 +5809,7 @@ "type": "tidelift" } ], - "time": "2022-02-25T10:48:52+00:00" + "time": "2022-03-31T17:18:25+00:00" }, { "name": "symfony/polyfill-intl-grapheme", @@ -6058,16 +6061,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.0.0", + "version": "v3.0.1", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603" + "reference": "e517458f278c2131ca9f262f8fbaf01410f2c65c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/36715ebf9fb9db73db0cb24263c79077c6fe8603", - "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/e517458f278c2131ca9f262f8fbaf01410f2c65c", + "reference": "e517458f278c2131ca9f262f8fbaf01410f2c65c", "shasum": "" }, "require": { @@ -6120,7 +6123,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.0.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.0.1" }, "funding": [ { @@ -6136,7 +6139,7 @@ "type": "tidelift" } ], - "time": "2021-11-04T17:53:12+00:00" + "time": "2022-03-13T20:10:05+00:00" }, { "name": "symfony/string", @@ -6324,16 +6327,16 @@ }, { "name": "twig/twig", - "version": "v3.3.8", + "version": "v3.3.10", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "972d8604a92b7054828b539f2febb0211dd5945c" + "reference": "8442df056c51b706793adf80a9fd363406dd3674" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/972d8604a92b7054828b539f2febb0211dd5945c", - "reference": "972d8604a92b7054828b539f2febb0211dd5945c", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8442df056c51b706793adf80a9fd363406dd3674", + "reference": "8442df056c51b706793adf80a9fd363406dd3674", "shasum": "" }, "require": { @@ -6384,7 +6387,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.3.8" + "source": "https://github.com/twigphp/Twig/tree/v3.3.10" }, "funding": [ { @@ -6396,7 +6399,7 @@ "type": "tidelift" } ], - "time": "2022-02-04T06:59:48+00:00" + "time": "2022-04-06T06:47:41+00:00" }, { "name": "vimeo/psalm", @@ -6580,5 +6583,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 2ddd5dd51b..ac990eef88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -370,6 +370,7 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -519,6 +520,11 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_ABUSE diff --git a/src/Appwrite/Exception/Certificate.php b/src/Appwrite/Exception/Certificate.php new file mode 100644 index 0000000000..9a146807d7 --- /dev/null +++ b/src/Appwrite/Exception/Certificate.php @@ -0,0 +1,38 @@ +incrementAttempts = $incrementAttempts; + + parent::__construct($message); + } + + /** + * Get value if attempts should be incremented. + * + * @return boolean + */ + public function getIncrementAttempts(): bool + { + return $this->incrementAttempts; + } + + /** + * Set if attempts should be incremented + * + * @param boolean $value + * + * @return void + */ + public function setIncrementAttempts(bool $value): void + { + $this->incrementAttempts = $value; + } +} \ No newline at end of file From 50412c7e806865f309daff1b0fd4ca1a6a3ada16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 11 Apr 2022 07:56:58 +0000 Subject: [PATCH 02/10] PR review changes --- app/tasks/maintenance.php | 4 +- app/tasks/ssl.php | 29 +++++++---- app/workers/certificates.php | 69 ++++++++++++++++---------- src/Appwrite/Exception/Certificate.php | 31 ------------ 4 files changed, 64 insertions(+), 69 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a50d8c4da9..e946daeb26 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -105,7 +105,7 @@ $cli $certificates = $dbForConsole->find('certificates', [ new Query('attempts', Query::TYPE_LESSER, [5]), // Maximum 5 attempts new Query('renewDate', Query::TYPE_LESSEREQUAL, [\time()]) // includes 60 days cooldown (we have 30 days to renew) - ], 300); // Limit 300 comes from LetsEncrypt (orders per 3 hours) + ], 200); // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) if(\count($certificates) > 0) { Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); @@ -134,7 +134,7 @@ $cli [$database, $returnDatabase] = getDatabase($register, '_console'); $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); + Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php index 6010c3fd1f..18f7ee8324 100644 --- a/app/tasks/ssl.php +++ b/app/tasks/ssl.php @@ -2,24 +2,35 @@ global $cli; +use Appwrite\Event\Event; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Validator\Hostname; $cli ->task('ssl') ->desc('Validate server certificates') - ->action(function () { - $domain = App::getEnv('_APP_DOMAIN', ''); + ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) + ->action(function ($domain) { + // HTTTP ping to check if domain is Appwrite server + $ch = \curl_init(); + \curl_setopt($ch, CURLOPT_URL, 'http://appwrite/manifest.json'); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_TIMEOUT, 5); + \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + \curl_exec($ch); + $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); + $error = \curl_error($ch); + \curl_close($ch); - // TODO: Instead of waiting, let's ping Traefik. If responds, we can schedule instantly - // TODO: Add support for argument (domain) + if($statusCode < 100) { + return Console::error('Appwrite connection refused with message: ' . $error); + } - Console::log('Issue a TLS certificate for master domain ('.$domain.') in 2 seconds. - Make sure your domain points to your server or restart to try again.'); + Console::success('Schedule a job to issue a TLS certificate for domain:' . $domain); - // Const for types not available here - ResqueScheduler::enqueueAt(\time() + 2, 'v1-certificates', 'CertificatesV1', [ + // Scheduje a job + Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ 'domain' => $domain, - 'skipRenewCheck' => true // TODO: Discuss this behabiour. true? false? parameter? How do we document it? ]); }); \ No newline at end of file diff --git a/app/workers/certificates.php b/app/workers/certificates.php index fea025d163..aca8519cc2 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -34,21 +34,38 @@ class CertificatesV1 extends Worker $certificate = new Document(); + + /** + * 1. Read arguments and validate domain + * 2. Get main domain + * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) + * 4. Validate renew date with certificate file, unless requested to skip by parameter + * 5. Validate security email. Cannot be empty, required by LetsEncrypt + * 6. Issue a certificate using certbot CLI + * 7. Update 'log' attribute on certificate document with Certbot message + * 8. Create storage folder for certificate, if not ready already + * 9. Move certificates from Certbot location to our Storage + * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 11. Read certificate file and update 'renewDate' on certificate document + * 12. Update 'issueDate' and 'attempts' on certificate + * + * If at any point unexpected error occurs, program stops without applying changes to document, and error is thrown into worker + * + * If code stops with expected error: + * 1. 'log' attribute on document is updated with error message + * 2. 'attempts' amount is increased + * 3. Console log is shown + * 4. Email is sent to security email + * + * Unless unexpected error occurs, at the end, we: + * 1. Update 'updated' attribute on document + * 2. Save document to database + * 3. Update all domains documents with current certificate ID + * + * Note: Renewals are checked and scheduled from maintenence worker + */ + try { - /** - * TODO: Update - * 1. Get new domain document - DONE - * 1.1. Validate domain is valid, public suffix is known and CNAME records are verified - DONE - * 2. Check if a certificate already exists - DONE - * 3. Check if certificate is about to expire, if not - skip it - * 3.1. Create / renew certificate - * 3.2. Update loadblancer - * 3.3. Update database (domains, change date, expiry) - * 3.4. Set retry on failure - * 3.5. Schedule to renew certificate in 60 days - */ - - // Get attributes $domain = $this->args['domain']; // String of domain (hostname) $domain = new Domain((!empty($domain)) ? $domain : ''); @@ -108,7 +125,7 @@ class CertificatesV1 extends Worker try { $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t']; + $validTo = $certData['validTo_time_t'] ?? 0; if (empty($validTo)) { throw new Exception('Invalid expiry date.'); @@ -146,7 +163,7 @@ class CertificatesV1 extends Worker // Unexpected error, usually 5XX, API limits, ... if ($exit !== 0) { - throw new ExceptionCertificate('Failed to issue a certificate with message: ' . $stderr, true); + throw new ExceptionCertificate('Failed to issue a certificate with message: ' . $stderr); } // Command succeeded, store all data into document @@ -161,25 +178,25 @@ class CertificatesV1 extends Worker $path = APP_STORAGE_CERTIFICATES . '/' . $domain->get(); if (!\is_readable($path)) { if (!\mkdir($path, 0755, true)) { - throw new ExceptionCertificate('Failed to create path for certificate.', true); + throw new ExceptionCertificate('Failed to create path for certificate.'); } } // Move generated files from certbot into our storage if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { - throw new ExceptionCertificate('Failed to rename certificate cert.pem: '.\json_encode($stdout), true); + throw new ExceptionCertificate('Failed to rename certificate cert.pem: '.\json_encode($stdout)); } if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/chain.pem')) { - throw new ExceptionCertificate('Failed to rename certificate chain.pem: ' . \json_encode($stdout), true); + throw new ExceptionCertificate('Failed to rename certificate chain.pem: ' . \json_encode($stdout)); } if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/fullchain.pem')) { - throw new ExceptionCertificate('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout), true); + throw new ExceptionCertificate('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout)); } if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/privkey.pem')) { - throw new ExceptionCertificate('Failed to rename certificate privkey.pem: ' . \json_encode($stdout), true); + throw new ExceptionCertificate('Failed to rename certificate privkey.pem: ' . \json_encode($stdout)); } // This multi-line syntax helps IDE @@ -191,14 +208,14 @@ class CertificatesV1 extends Worker // Save configuration into Traefik using our new cert files if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain->get() . '.yml', $config)) { - throw new ExceptionCertificate('Failed to save Traefik configuration.', true); + throw new ExceptionCertificate('Failed to save Traefik configuration.'); } // Read new renew date from cert file // TODO: This might not be required, we could calculate it. But this feels safer $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/cert.pem'; $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t']; + $validTo = $certData['validTo_time_t'] ?? 0; $expiryInAdvance = (60*60*24*30); $certificate->setAttribute('renewDate', $validTo - $expiryInAdvance); @@ -218,10 +235,8 @@ class CertificatesV1 extends Worker $attempt = $certificate->getAttribute('attempts', 0); $attempt++; - // Save increased attempts count if requested by exception - if($e->getIncrementAttempts()) { - $certificate->setAttribute('attempts', $attempt); - } + // Save increased attempts count + $certificate->setAttribute('attempts', $attempt); Console::warning('Cannot renew domain (' . $domain->get() . ') on attempt no. ' . $attempt . ' certificate: ' . $e->getMessage()); } finally { diff --git a/src/Appwrite/Exception/Certificate.php b/src/Appwrite/Exception/Certificate.php index 9a146807d7..0b044015d4 100644 --- a/src/Appwrite/Exception/Certificate.php +++ b/src/Appwrite/Exception/Certificate.php @@ -4,35 +4,4 @@ namespace Appwrite\Exception; class Certificate extends \Exception { - private $incrementAttempts = false; - - // Only set incrementAttempts to TRUE if exception occured AFTER certbot command execution - public function __construct(string $message, bool $incrementAttempts = false) - { - $this->incrementAttempts = $incrementAttempts; - - parent::__construct($message); - } - - /** - * Get value if attempts should be incremented. - * - * @return boolean - */ - public function getIncrementAttempts(): bool - { - return $this->incrementAttempts; - } - - /** - * Set if attempts should be incremented - * - * @param boolean $value - * - * @return void - */ - public function setIncrementAttempts(bool $value): void - { - $this->incrementAttempts = $value; - } } \ No newline at end of file From 7159070067be8e49216ff088f3e8c8f014716d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 11 Apr 2022 08:34:11 +0000 Subject: [PATCH 03/10] PR review changes --- app/config/locale/translations/en.json | 6 ++++++ app/init.php | 1 + app/workers/certificates.php | 28 +++++++++++++++++++++++++- app/workers/mails.php | 12 +++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index c00740b130..e533d82795 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -27,6 +27,12 @@ "emails.invitation.footer": "If you are not interested, you can ignore this message.", "emails.invitation.thanks": "Thanks", "emails.invitation.signature": "{{project}} team", + "emails.certificate.subject": "Certificate failure for %s", + "emails.certificate.hello": "Hello", + "emails.certificate.body": "Certificate for your domain '{{domain}}' could not be renewed. This is attempt no. {{attempt}}, and the failure was caused by: {{error}}", + "emails.certificate.footer": "Certificate will still be valid for 30 days since first failure. We highly recommend investigating the case, otherwise your domain will end up without a secure certificate.", + "emails.certificate.thanks": "Thanks", + "emails.certificate.signature": "{{project}} team", "locale.country.unknown": "Unknown", "countries.af": "Afghanistan", "countries.ao": "Angola", diff --git a/app/init.php b/app/init.php index fd509297f6..1c8dc5aadc 100644 --- a/app/init.php +++ b/app/init.php @@ -127,6 +127,7 @@ const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; const MAIL_TYPE_RECOVERY = 'recovery'; const MAIL_TYPE_INVITATION = 'invitation'; +const MAIL_TYPE_CERTIFICATE = 'certificat'; // Auth Types const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; diff --git a/app/workers/certificates.php b/app/workers/certificates.php index aca8519cc2..825bb1689c 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -239,6 +239,21 @@ class CertificatesV1 extends Worker $certificate->setAttribute('attempts', $attempt); Console::warning('Cannot renew domain (' . $domain->get() . ') on attempt no. ' . $attempt . ' certificate: ' . $e->getMessage()); + + // Send email to security email + Resque::enqueue(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME, [ + 'from' => 'console', + 'project' => 'console', + 'name' => 'Appwrite Administrator', + 'recipient' => App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'), + 'url' => 'https://' . $domain->get(), + 'locale' => App::getEnv('_APP_LOCALE', 'en'), + 'type' => MAIL_TYPE_CERTIFICATE, + + 'domain' => $domain->get(), + 'error' => $e->getMessage(), + 'attempt' => $attempt + ]); } finally { // All actions result in new updatedAt date $certificate->setAttribute('updated', \time()); @@ -257,7 +272,18 @@ class CertificatesV1 extends Worker // Update domains with new certificate ID $certificateId = $certificate->getId(); - // TODO: Add logic for updating domains + + $domains = $dbForConsole->find('domains', [ + new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) + ], 1000); + + foreach ($domains as $domainDocument) { + $domainDocument->setAttribute('updated', \time()); + $domainDocument->setAttribute('certificateId', $certificateId); + + $dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument); + $dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId')); + } Authorization::reset(); } diff --git a/app/workers/mails.php b/app/workers/mails.php index 25ddb438c0..6905593a8c 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -46,6 +46,16 @@ class MailsV1 extends Worker $body = Template::fromFile(__DIR__ . '/../config/locale/templates/email-base.tpl'); $subject = ''; switch ($type) { + case MAIL_TYPE_CERTIFICATE: + $domain = $this->args['domain']; + $error = $this->args['error']; + $attempt = $this->args['attempt']; + + $subject = \sprintf($locale->getText("$prefix.subject"), $domain); + $body->setParam('{{domain}}', $domain); + $body->setParam('{{error}}', $error); + $body->setParam('{{attempt}}', $attempt); + break; case MAIL_TYPE_INVITATION: $subject = \sprintf($locale->getText("$prefix.subject"), $this->args['team'], $project); $body->setParam('{{owner}}', $this->args['owner']); @@ -126,6 +136,8 @@ class MailsV1 extends Worker switch ($type) { case MAIL_TYPE_RECOVERY: return 'emails.recovery'; + case MAIL_TYPE_CERTIFICATE: + return 'emails.certificate'; case MAIL_TYPE_INVITATION: return 'emails.invitation'; case MAIL_TYPE_VERIFICATION: From 99537545ced33d88295d5c6ce78e3715f6628709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 12 Apr 2022 11:46:43 +0000 Subject: [PATCH 04/10] Update lockfile after merge --- composer.lock | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/composer.lock b/composer.lock index be60c6c07f..9bafae75a0 100644 --- a/composer.lock +++ b/composer.lock @@ -300,16 +300,16 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.12.2", + "version": "v1.13.0", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "77e6ede2e01c4cfaade114fe1e07d2f9756949f1" + "reference": "afec8e58ec93d2291c127fa19709a048f28641e5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/77e6ede2e01c4cfaade114fe1e07d2f9756949f1", - "reference": "77e6ede2e01c4cfaade114fe1e07d2f9756949f1", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/afec8e58ec93d2291c127fa19709a048f28641e5", + "reference": "afec8e58ec93d2291c127fa19709a048f28641e5", "shasum": "" }, "require": { @@ -341,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.12.2" + "source": "https://github.com/colinmollenhour/credis/tree/v1.13.0" }, - "time": "2022-03-08T18:12:43+00:00" + "time": "2022-04-07T14:57:22+00:00" }, { "name": "composer/package-versions-deprecated", @@ -2250,16 +2250,16 @@ }, { "name": "utopia-php/framework", - "version": "0.19.7", + "version": "0.19.8", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "f17afe77a21873b9be18ebc05283813468b4283a" + "reference": "8c3b3e330546fd6cd65bd1f8d8d08882ff3abb7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/f17afe77a21873b9be18ebc05283813468b4283a", - "reference": "f17afe77a21873b9be18ebc05283813468b4283a", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/8c3b3e330546fd6cd65bd1f8d8d08882ff3abb7d", + "reference": "8c3b3e330546fd6cd65bd1f8d8d08882ff3abb7d", "shasum": "" }, "require": { @@ -2293,9 +2293,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.7" + "source": "https://github.com/utopia-php/framework/tree/0.19.8" }, - "time": "2022-02-18T00:04:49+00:00" + "time": "2022-04-12T00:28:15+00:00" }, { "name": "utopia-php/image", @@ -6327,16 +6327,16 @@ }, { "name": "twig/twig", - "version": "v3.3.9", + "version": "v3.3.10", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "6ff9b0e440fa66f97f207e181c41340ddfa5683d" + "reference": "8442df056c51b706793adf80a9fd363406dd3674" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/6ff9b0e440fa66f97f207e181c41340ddfa5683d", - "reference": "6ff9b0e440fa66f97f207e181c41340ddfa5683d", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8442df056c51b706793adf80a9fd363406dd3674", + "reference": "8442df056c51b706793adf80a9fd363406dd3674", "shasum": "" }, "require": { @@ -6387,7 +6387,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.3.9" + "source": "https://github.com/twigphp/Twig/tree/v3.3.10" }, "funding": [ { @@ -6399,7 +6399,7 @@ "type": "tidelift" } ], - "time": "2022-03-25T09:37:52+00:00" + "time": "2022-04-06T06:47:41+00:00" }, { "name": "vimeo/psalm", From e49d6fd7661463d23f80c5bca8670a3c7c185527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 13 Apr 2022 12:26:07 +0000 Subject: [PATCH 05/10] Review changes --- app/init.php | 2 +- app/realtime.php | 10 +++++----- app/tasks/maintenance.php | 7 ++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/init.php b/app/init.php index 1c8dc5aadc..221742fc2f 100644 --- a/app/init.php +++ b/app/init.php @@ -127,7 +127,7 @@ const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; const MAIL_TYPE_RECOVERY = 'recovery'; const MAIL_TYPE_INVITATION = 'invitation'; -const MAIL_TYPE_CERTIFICATE = 'certificat'; +const MAIL_TYPE_CERTIFICATE = 'certificate'; // Auth Types const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; diff --git a/app/realtime.php b/app/realtime.php index 8d36069edb..275305e479 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -93,7 +93,7 @@ $logError = function(Throwable $error, string $action) use ($register) { $server->error($logError); -function getDatabase(Registry &$register, string $namespace) +function getConsoleDB(Registry &$register, string $namespace) { $attempts = 0; @@ -141,7 +141,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume */ go(function () use ($register, $containerId, &$statsDocument, $logError) { try { - [$database, $returnDatabase] = getDatabase($register, '_console'); + [$database, $returnDatabase] = getConsoleDB($register, '_console'); $document = new Document([ '$id' => $database->getId(), '$collection' => 'realtime', @@ -194,7 +194,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } try { - [$database, $returnDatabase] = getDatabase($register, '_console'); + [$database, $returnDatabase] = getConsoleDB($register, '_console'); $statsDocument ->setAttribute('timestamp', time()) @@ -221,7 +221,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, */ if ($realtime->hasSubscriber('console', 'role:member', 'project')) { - [$database, $returnDatabase] = getDatabase($register, '_console'); + [$database, $returnDatabase] = getConsoleDB($register, '_console'); $payload = []; @@ -325,7 +325,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, return; } - [$database, $returnDatabase] = getDatabase($register, "_{$projectId}"); + [$database, $returnDatabase] = getConsoleDB($register, "_{$projectId}"); $user = $database->getDocument('users', $userId); diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index e946daeb26..db2d2e808a 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -13,7 +13,7 @@ use Utopia\Registry\Registry; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Database\Query; -function getDatabase(Registry &$register) +function getConsoleDB(Registry &$register) { $attempts = 0; @@ -103,7 +103,7 @@ $cli /** @var Utopia\Database\Database $dbForConsole */ $certificates = $dbForConsole->find('certificates', [ - new Query('attempts', Query::TYPE_LESSER, [5]), // Maximum 5 attempts + new Query('attempts', Query::TYPE_LESSEREQUAL, [5]), // Maximum 5 attempts new Query('renewDate', Query::TYPE_LESSEREQUAL, [\time()]) // includes 60 days cooldown (we have 30 days to renew) ], 200); // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) @@ -131,7 +131,8 @@ $cli Console::loop(function() use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { go(function () use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { try { - [$database, $returnDatabase] = getDatabase($register, '_console'); + [$database, $returnDatabase] = getConsoleDB + ($register, '_console'); $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); From a32713011f3b459798d935ddc3b4e9321a1e15aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 27 Apr 2022 09:02:31 +0000 Subject: [PATCH 06/10] PR review changes --- app/tasks/maintenance.php | 3 +-- app/tasks/ssl.php | 17 +---------------- app/workers/certificates.php | 32 +++++++++++++++++--------------- composer.lock | 34 +++++++++++++--------------------- 4 files changed, 32 insertions(+), 54 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index db2d2e808a..087b16cf95 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -131,8 +131,7 @@ $cli Console::loop(function() use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { go(function () use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { try { - [$database, $returnDatabase] = getConsoleDB - ($register, '_console'); + [$database, $returnDatabase] = getConsoleDB($register, '_console'); $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php index 18f7ee8324..e1724e8401 100644 --- a/app/tasks/ssl.php +++ b/app/tasks/ssl.php @@ -12,22 +12,7 @@ $cli ->desc('Validate server certificates') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) ->action(function ($domain) { - // HTTTP ping to check if domain is Appwrite server - $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, 'http://appwrite/manifest.json'); - \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - \curl_setopt($ch, CURLOPT_TIMEOUT, 5); - \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - \curl_exec($ch); - $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); - $error = \curl_error($ch); - \curl_close($ch); - - if($statusCode < 100) { - return Console::error('Appwrite connection refused with message: ' . $error); - } - - Console::success('Schedule a job to issue a TLS certificate for domain:' . $domain); + Console::success('Scheduling a job to issue a TLS certificate for domain:' . $domain); // Scheduje a job Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 825bb1689c..25eda60a91 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -18,12 +18,15 @@ Console::success(APP_NAME . ' certificates worker v1 has started'); class CertificatesV1 extends Worker { + private $certificate = null; // run function fills this. onError callback uses it + public function getName(): string { return "certificates"; } public function init(): void { + } public function run(): void @@ -32,8 +35,7 @@ class CertificatesV1 extends Worker $dbForConsole = $this->getConsoleDB(); - $certificate = new Document(); - + $this->certificate = new Document(); /** * 1. Read arguments and validate domain @@ -70,7 +72,7 @@ class CertificatesV1 extends Worker $domain = $this->args['domain']; // String of domain (hostname) $domain = new Domain((!empty($domain)) ? $domain : ''); - $certificate->setAttribute('domain', $domain->get()); + $this->certificate->setAttribute('domain', $domain->get()); $skipRenewCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file @@ -169,7 +171,7 @@ class CertificatesV1 extends Worker // Command succeeded, store all data into document // We store stderr too, because it may include warnings // This is only stored if everytng below passes too. Otherwise, it will be overwritten by error message - $certificate->setAttribute('log', \json_encode([ + $this->certificate->setAttribute('log', \json_encode([ 'stdout' => $stdout, 'stderr' => $stderr, ])); @@ -217,26 +219,26 @@ class CertificatesV1 extends Worker $certData = openssl_x509_parse(file_get_contents($certPath)); $validTo = $certData['validTo_time_t'] ?? 0; $expiryInAdvance = (60*60*24*30); - $certificate->setAttribute('renewDate', $validTo - $expiryInAdvance); + $this->certificate->setAttribute('renewDate', $validTo - $expiryInAdvance); // All went well at this point 🥳 // Reset attempts count for next renwal - $certificate->setAttribute('attempts', 0); + $this->certificate->setAttribute('attempts', 0); // Mark issue date - $certificate->setAttribute('issueDate', \time()); + $this->certificate->setAttribute('issueDate', \time()); } catch(ExceptionCertificate $e) { // These exceptions are expected if renew shouldn't or can't happen // Add exception as log into certificate - $certificate->setAttribute('log', $e->getMessage()); + $this->certificate->setAttribute('log', $e->getMessage()); - $attempt = $certificate->getAttribute('attempts', 0); + $attempt = $this->certificate->getAttribute('attempts', 0); $attempt++; // Save increased attempts count - $certificate->setAttribute('attempts', $attempt); + $this->certificate->setAttribute('attempts', $attempt); Console::warning('Cannot renew domain (' . $domain->get() . ') on attempt no. ' . $attempt . ' certificate: ' . $e->getMessage()); @@ -256,22 +258,22 @@ class CertificatesV1 extends Worker ]); } finally { // All actions result in new updatedAt date - $certificate->setAttribute('updated', \time()); + $this->certificate->setAttribute('updated', \time()); // Save certificate data into database // Check if update or insert required $certificateDocument = $dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) ]); if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { // Merge new data with current data - $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); + $this->certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $this->certificate->getArrayCopy())); - $certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); + $this->certificate = $dbForConsole->updateDocument('certificates', $this->certificate->getId(), $this->certificate); } else { - $certificate = $dbForConsole->createDocument('certificates', $certificate); + $this->certificate = $dbForConsole->createDocument('certificates', $this->certificate); } // Update domains with new certificate ID - $certificateId = $certificate->getId(); + $certificateId = $this->certificate->getId(); $domains = $dbForConsole->find('domains', [ new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) diff --git a/composer.lock b/composer.lock index 9bafae75a0..c82900e1bb 100644 --- a/composer.lock +++ b/composer.lock @@ -2250,16 +2250,16 @@ }, { "name": "utopia-php/framework", - "version": "0.19.8", + "version": "0.19.20", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "8c3b3e330546fd6cd65bd1f8d8d08882ff3abb7d" + "reference": "65ced168db8f6e188ceeb0d101f57552c3d8b2af" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/8c3b3e330546fd6cd65bd1f8d8d08882ff3abb7d", - "reference": "8c3b3e330546fd6cd65bd1f8d8d08882ff3abb7d", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/65ced168db8f6e188ceeb0d101f57552c3d8b2af", + "reference": "65ced168db8f6e188ceeb0d101f57552c3d8b2af", "shasum": "" }, "require": { @@ -2293,9 +2293,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.8" + "source": "https://github.com/utopia-php/framework/tree/0.19.20" }, - "time": "2022-04-12T00:28:15+00:00" + "time": "2022-04-14T15:42:37+00:00" }, { "name": "utopia-php/image", @@ -3550,16 +3550,16 @@ }, { "name": "matthiasmullie/minify", - "version": "1.3.66", + "version": "1.3.67", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "45fd3b0f1dfa2c965857c6d4a470bea52adc31a6" + "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/45fd3b0f1dfa2c965857c6d4a470bea52adc31a6", - "reference": "45fd3b0f1dfa2c965857c6d4a470bea52adc31a6", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", + "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", "shasum": "" }, "require": { @@ -3608,23 +3608,15 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.66" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.67" }, "funding": [ { - "url": "https://github.com/[user1", - "type": "github" - }, - { - "url": "https://github.com/matthiasmullie] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g.", - "type": "github" - }, - { - "url": "https://github.com/user2", + "url": "https://github.com/matthiasmullie", "type": "github" } ], - "time": "2021-01-06T15:18:10+00:00" + "time": "2022-03-24T08:54:59+00:00" }, { "name": "matthiasmullie/path-converter", From e4379ec850d8c9117c815c83b98083f11705d4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 27 Apr 2022 09:42:15 +0000 Subject: [PATCH 07/10] PR review changes --- app/workers/certificates.php | 31 +++++++++++++------------- src/Appwrite/Exception/Certificate.php | 7 ------ 2 files changed, 15 insertions(+), 23 deletions(-) delete mode 100644 src/Appwrite/Exception/Certificate.php diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 25eda60a91..6c246d0af8 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -3,7 +3,6 @@ use Appwrite\Event\Event; use Appwrite\Network\Validator\CNAME; use Appwrite\Resque\Worker; -use Appwrite\Exception\Certificate as ExceptionCertificate; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Document; @@ -91,11 +90,11 @@ class CertificatesV1 extends Worker } if (empty($domain->get())) { - throw new ExceptionCertificate('Missing certificate domain.'); + throw new Exception('Missing certificate domain.'); } if (!$domain->isKnown() || $domain->isTest()) { - throw new ExceptionCertificate('Unknown public suffix for domain.'); + throw new Exception('Unknown public suffix for domain.'); } if ($validateCNAME) { @@ -105,13 +104,13 @@ class CertificatesV1 extends Worker $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); if (!$target->isKnown() || $target->isTest()) { - throw new ExceptionCertificate('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); + throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); } // Verify domain with DNS records $validator = new CNAME($target->get()); if (!$validator->isValid($domain->get())) { - throw new ExceptionCertificate('Failed to verify domain DNS records.'); + throw new Exception('Failed to verify domain DNS records.'); } } else { // Main domain validation @@ -133,21 +132,21 @@ class CertificatesV1 extends Worker throw new Exception('Invalid expiry date.'); } } catch(\Throwable $th) { - throw new ExceptionCertificate('Unable to read certificate file (cert.pem).'); + throw new Exception('Unable to read certificate file (cert.pem).'); } // LetsEncrypt allows renewal 30 days before expiry $expiryInAdvance = (60*60*24*30); if ($validTo - $expiryInAdvance > \time()) { $validToVerbose = date('d-m-Y H:i:s', $validTo); - throw new ExceptionCertificate('Renew isn\'t required. Next renew at ' . $validToVerbose); + throw new Exception('Renew isn\'t required. Next renew at ' . $validToVerbose); } } // Email for alerts is required by LetsEncrypt $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); if (empty($email)) { - throw new ExceptionCertificate('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); + throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); } // LetsEncrypt communication to issue certificate (using certbot CLI) @@ -165,7 +164,7 @@ class CertificatesV1 extends Worker // Unexpected error, usually 5XX, API limits, ... if ($exit !== 0) { - throw new ExceptionCertificate('Failed to issue a certificate with message: ' . $stderr); + throw new Exception('Failed to issue a certificate with message: ' . $stderr); } // Command succeeded, store all data into document @@ -180,25 +179,25 @@ class CertificatesV1 extends Worker $path = APP_STORAGE_CERTIFICATES . '/' . $domain->get(); if (!\is_readable($path)) { if (!\mkdir($path, 0755, true)) { - throw new ExceptionCertificate('Failed to create path for certificate.'); + throw new Exception('Failed to create path for certificate.'); } } // Move generated files from certbot into our storage if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { - throw new ExceptionCertificate('Failed to rename certificate cert.pem: '.\json_encode($stdout)); + throw new Exception('Failed to rename certificate cert.pem: '.\json_encode($stdout)); } if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/chain.pem')) { - throw new ExceptionCertificate('Failed to rename certificate chain.pem: ' . \json_encode($stdout)); + throw new Exception('Failed to rename certificate chain.pem: ' . \json_encode($stdout)); } if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/fullchain.pem')) { - throw new ExceptionCertificate('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout)); + throw new Exception('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout)); } if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/privkey.pem')) { - throw new ExceptionCertificate('Failed to rename certificate privkey.pem: ' . \json_encode($stdout)); + throw new Exception('Failed to rename certificate privkey.pem: ' . \json_encode($stdout)); } // This multi-line syntax helps IDE @@ -210,7 +209,7 @@ class CertificatesV1 extends Worker // Save configuration into Traefik using our new cert files if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain->get() . '.yml', $config)) { - throw new ExceptionCertificate('Failed to save Traefik configuration.'); + throw new Exception('Failed to save Traefik configuration.'); } // Read new renew date from cert file @@ -228,7 +227,7 @@ class CertificatesV1 extends Worker // Mark issue date $this->certificate->setAttribute('issueDate', \time()); - } catch(ExceptionCertificate $e) { + } catch(Throwable $e) { // These exceptions are expected if renew shouldn't or can't happen // Add exception as log into certificate diff --git a/src/Appwrite/Exception/Certificate.php b/src/Appwrite/Exception/Certificate.php deleted file mode 100644 index 0b044015d4..0000000000 --- a/src/Appwrite/Exception/Certificate.php +++ /dev/null @@ -1,7 +0,0 @@ - Date: Wed, 11 May 2022 13:11:58 +0000 Subject: [PATCH 08/10] PR review changes --- app/realtime.php | 10 +- app/tasks/maintenance.php | 61 ++-- app/tasks/ssl.php | 1 + app/views/install/compose.phtml | 2 +- app/workers/certificates.php | 506 +++++++++++++++++++------------- composer.lock | 36 +-- 6 files changed, 352 insertions(+), 264 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 275305e479..8d36069edb 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -93,7 +93,7 @@ $logError = function(Throwable $error, string $action) use ($register) { $server->error($logError); -function getConsoleDB(Registry &$register, string $namespace) +function getDatabase(Registry &$register, string $namespace) { $attempts = 0; @@ -141,7 +141,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume */ go(function () use ($register, $containerId, &$statsDocument, $logError) { try { - [$database, $returnDatabase] = getConsoleDB($register, '_console'); + [$database, $returnDatabase] = getDatabase($register, '_console'); $document = new Document([ '$id' => $database->getId(), '$collection' => 'realtime', @@ -194,7 +194,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } try { - [$database, $returnDatabase] = getConsoleDB($register, '_console'); + [$database, $returnDatabase] = getDatabase($register, '_console'); $statsDocument ->setAttribute('timestamp', time()) @@ -221,7 +221,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, */ if ($realtime->hasSubscriber('console', 'role:member', 'project')) { - [$database, $returnDatabase] = getConsoleDB($register, '_console'); + [$database, $returnDatabase] = getDatabase($register, '_console'); $payload = []; @@ -325,7 +325,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, return; } - [$database, $returnDatabase] = getConsoleDB($register, "_{$projectId}"); + [$database, $returnDatabase] = getDatabase($register, "_{$projectId}"); $user = $database->getDocument('users', $userId); diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 087b16cf95..056ee59fcd 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -9,26 +9,22 @@ use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; -use Utopia\Registry\Registry; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Database\Query; -function getConsoleDB(Registry &$register) +function getConsoleDB(): Database { + global $register; + $attempts = 0; do { try { $attempts++; - - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); + $cache = new Cache(new RedisCache($register->get('cache'))); + $database = new Database(new MariaDB($register->get('db')), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); - + $database->setNamespace('_console'); // Main DB break; // leave loop if successful } catch(\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); @@ -39,20 +35,13 @@ function getConsoleDB(Registry &$register) } } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - return [ - $database, - function () use ($register, $db, $redis) { - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - } - ]; - -}; + return $database; +} $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') - ->action(function () use ($register) { + ->action(function () { Console::title('Maintenance V1'); Console::success(APP_NAME.' maintenance process v1 has started'); @@ -128,25 +117,17 @@ $cli $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { - go(function () use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { - try { - [$database, $returnDatabase] = getConsoleDB($register, '_console'); - - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); - notifyDeleteExecutionLogs($executionLogsRetention); - notifyDeleteAbuseLogs($abuseLogsRetention); - notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); - notifyDeleteConnections(); - - renewCertificates($database); - } catch (\Throwable $th) { - throw $th; - } finally { - call_user_func($returnDatabase); - } - }); + Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { + $database = getConsoleDB(); + + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); + notifyDeleteExecutionLogs($executionLogsRetention); + notifyDeleteAbuseLogs($abuseLogsRetention); + notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); + notifyDeleteConnections(); + + renewCertificates($database); }, $interval); }); \ No newline at end of file diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php index e1724e8401..f28c0b8532 100644 --- a/app/tasks/ssl.php +++ b/app/tasks/ssl.php @@ -17,5 +17,6 @@ $cli // Scheduje a job Resque::enqueue(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, [ 'domain' => $domain, + 'skipRenewCheck' => true ]); }); \ No newline at end of file diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index c1c28d2e2c..470efc8750 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -339,6 +339,7 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -471,7 +472,6 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 6c246d0af8..0b6a45f0c3 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -17,7 +17,13 @@ Console::success(APP_NAME . ' certificates worker v1 has started'); class CertificatesV1 extends Worker { - private $certificate = null; // run function fills this. onError callback uses it + + /** + * Database connection shared across all methods of this file + * + * @var Database + */ + private $dbForConsole; public function getName(): string { return "certificates"; @@ -32,16 +38,14 @@ class CertificatesV1 extends Worker { Authorization::disable(); - $dbForConsole = $this->getConsoleDB(); - - $this->certificate = new Document(); + $this->dbForConsole = $this->getConsoleDB(); /** * 1. Read arguments and validate domain * 2. Get main domain * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) - * 4. Validate renew date with certificate file, unless requested to skip by parameter - * 5. Validate security email. Cannot be empty, required by LetsEncrypt + * 4. Validate security email. Cannot be empty, required by LetsEncrypt + * 5. Validate renew date with certificate file, unless requested to skip by parameter * 6. Issue a certificate using certbot CLI * 7. Update 'log' attribute on certificate document with Certbot message * 8. Create storage folder for certificate, if not ready already @@ -67,225 +71,71 @@ class CertificatesV1 extends Worker */ try { - // Get attributes + // Read arguments $domain = $this->args['domain']; // String of domain (hostname) + $skipRenewCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file + $domain = new Domain((!empty($domain)) ? $domain : ''); - $this->certificate->setAttribute('domain', $domain->get()); - - $skipRenewCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file - - $mainDomain = null; // ENV or first ever visited domain - if (!empty(App::getEnv('_APP_DOMAIN', ''))) { - $mainDomain = App::getEnv('_APP_DOMAIN', ''); - } else { - $domainDocument = $dbForConsole->findOne('domains', [], 0, ['_id'], ['ASC']); - $mainDomain = $domainDocument ? $domainDocument->getAttribute('domain') : $domain->get(); - } - - // If not main domain, we will check CNAME record - $validateCNAME = false; - if ($domain->get() !== $mainDomain) { - $validateCNAME = true; - } - - if (empty($domain->get())) { - throw new Exception('Missing certificate domain.'); - } - - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain.'); - } - - if ($validateCNAME) { - // TODO: Would be awesome to also support A/AAAA records here. Maybe dry run? - - // Validate if domain target is properly configured - $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); - - if (!$target->isKnown() || $target->isTest()) { - throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); - } - - // Verify domain with DNS records - $validator = new CNAME($target->get()); - if (!$validator->isValid($domain->get())) { - throw new Exception('Failed to verify domain DNS records.'); - } - } else { - // Main domain validation - // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + // Get current certificate + $certificate = $this->dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) ]); + + // If we don't have certificate for domain yet, let's create new document. At the end we save it + if(!$certificate) { + $certificate = new Document(); + $certificate->setAttribute('domain', $domain->get()); } - // If certificate exists already, double-check expiry date - // If asked to skip, we won't - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/cert.pem'; - if (!$skipRenewCheck && \file_exists($certPath)) { - $validTo = null; - - try { - $certData = openssl_x509_parse(file_get_contents($certPath)); - - $validTo = $certData['validTo_time_t'] ?? 0; - - if (empty($validTo)) { - throw new Exception('Invalid expiry date.'); - } - } catch(\Throwable $th) { - throw new Exception('Unable to read certificate file (cert.pem).'); - } - - // LetsEncrypt allows renewal 30 days before expiry - $expiryInAdvance = (60*60*24*30); - if ($validTo - $expiryInAdvance > \time()) { - $validToVerbose = date('d-m-Y H:i:s', $validTo); - throw new Exception('Renew isn\'t required. Next renew at ' . $validToVerbose); - } - } - // Email for alerts is required by LetsEncrypt $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); if (empty($email)) { throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); } - - // LetsEncrypt communication to issue certificate (using certbot CLI) - $stdout = ''; - $stderr = ''; - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain->get()}", '', $stdout, $stderr); - // All exceptions from now on will be marked to increment attempts count. This allows us to only limit attempts for domains that failed on LectEncrypt side. - // Such attempts count allows us to prevent API limit abuse with always failing domains - - // Unexpected error, usually 5XX, API limits, ... - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); + $mainDomain = $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + $this->validateDomain($domain, $isMainDomain); + + // If certificate exists already, double-check expiry date + // If asked to skip, we won't + if(!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { + throw new Exception('Renew isn\'t required.'); } + // Generate certificate files using Let's Encrypt + $letsEncryptData = $this->letsEncryptAction($domain->get(), $email); + // Command succeeded, store all data into document // We store stderr too, because it may include warnings - // This is only stored if everytng below passes too. Otherwise, it will be overwritten by error message - $this->certificate->setAttribute('log', \json_encode([ - 'stdout' => $stdout, - 'stderr' => $stderr, + $certificate->setAttribute('log', \json_encode([ + 'stdout' => $letsEncryptData['stdout'], + 'stderr' => $letsEncryptData['stderr'], ])); - // Prepare folder in storage for domain - $path = APP_STORAGE_CERTIFICATES . '/' . $domain->get(); - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path for certificate.'); - } - } - - // Move generated files from certbot into our storage - if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem: '.\json_encode($stdout)); - } - - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem: ' . \json_encode($stdout)); - } - - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem: ' . \json_encode($stdout)); - } - - if (!@\rename('/etc/letsencrypt/live/' . $domain->get() . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem: ' . \json_encode($stdout)); - } - - // This multi-line syntax helps IDE - $config = - "tls:" . - " certificates:" . - " - certFile: /storage/certificates/{$domain->get()}/fullchain.pem" . - " keyFile: /storage/certificates/{$domain->get()}/privkey.pem"; + // Give certificates to Traefik + $this->applyCertificateFiles($domain->get()); - // Save configuration into Traefik using our new cert files - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain->get() . '.yml', $config)) { - throw new Exception('Failed to save Traefik configuration.'); - } - - // Read new renew date from cert file - // TODO: This might not be required, we could calculate it. But this feels safer - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain->get() . '/cert.pem'; - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? 0; - $expiryInAdvance = (60*60*24*30); - $this->certificate->setAttribute('renewDate', $validTo - $expiryInAdvance); - - // All went well at this point 🥳 - - // Reset attempts count for next renwal - $this->certificate->setAttribute('attempts', 0); - - // Mark issue date - $this->certificate->setAttribute('issueDate', \time()); + // Update certificate info stored in database + $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('attempts', 0); + $certificate->setAttribute('issueDate', \time()); } catch(Throwable $e) { - // These exceptions are expected if renew shouldn't or can't happen + // Set exception as log in certificate document + $certificate->setAttribute('log', $e->getMessage()); - // Add exception as log into certificate - $this->certificate->setAttribute('log', $e->getMessage()); - - $attempt = $this->certificate->getAttribute('attempts', 0); - $attempt++; - - // Save increased attempts count - $this->certificate->setAttribute('attempts', $attempt); - - Console::warning('Cannot renew domain (' . $domain->get() . ') on attempt no. ' . $attempt . ' certificate: ' . $e->getMessage()); + // Increase attempts count + $attempts = $certificate->getAttribute('attempts', 0) + 1; + $certificate->setAttribute('attempts', $attempts); // Send email to security email - Resque::enqueue(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME, [ - 'from' => 'console', - 'project' => 'console', - 'name' => 'Appwrite Administrator', - 'recipient' => App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'), - 'url' => 'https://' . $domain->get(), - 'locale' => App::getEnv('_APP_LOCALE', 'en'), - 'type' => MAIL_TYPE_CERTIFICATE, - - 'domain' => $domain->get(), - 'error' => $e->getMessage(), - 'attempt' => $attempt - ]); + $this->notifyError($domain->get(), $e->getMessage(), $attempts); } finally { // All actions result in new updatedAt date - $this->certificate->setAttribute('updated', \time()); - - // Save certificate data into database - // Check if update or insert required - $certificateDocument = $dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) ]); - if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { - // Merge new data with current data - $this->certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $this->certificate->getArrayCopy())); - - $this->certificate = $dbForConsole->updateDocument('certificates', $this->certificate->getId(), $this->certificate); - } else { - $this->certificate = $dbForConsole->createDocument('certificates', $this->certificate); - } - - // Update domains with new certificate ID - $certificateId = $this->certificate->getId(); - - $domains = $dbForConsole->find('domains', [ - new Query('domain', Query::TYPE_EQUAL, [$domain->get()]) - ], 1000); - - foreach ($domains as $domainDocument) { - $domainDocument->setAttribute('updated', \time()); - $domainDocument->setAttribute('certificateId', $certificateId); - - $dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument); - $dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId')); - } + $certificate->setAttribute('updated', \time()); + // Save all changes we made to certificate document into database + $this->saveCertificateDocument($domain->get(), $certificate); + Authorization::reset(); } } @@ -293,4 +143,260 @@ class CertificatesV1 extends Worker public function shutdown(): void { } + + /** + * Save certificate data into database. + * + * @param string $domain Domain name that certificate is for + * @param Document $certificate Certificate document that we need to save + * + * @return void + */ + private function saveCertificateDocument(string $domain, Document $certificate) { + // Check if update or insert required + $certificateDocument = $this->dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain]) ]); + if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { + // Merge new data with current data + $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); + + $certificate = $this->dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); + } else { + $certificate = $this->dbForConsole->createDocument('certificates', $certificate); + } + + $certificateId = $certificate->getId(); + $this->updateDomainDocuments($certificateId, $domain); + } + + /** + * Get main domain. Needed as we do different checks for main and non-main domains. + * + * @return null|string Returns main domain. If null, there is no main domain yet. + */ + private function getMainDomain(): ?string { + if (!empty(App::getEnv('_APP_DOMAIN', ''))) { + $mainDomain = App::getEnv('_APP_DOMAIN', ''); + } else { + $domainDocument = $this->dbForConsole->findOne('domains', [], 0, ['_id'], ['ASC']); + if($domainDocument) { + $mainDomain = $domainDocument->getAttribute('domain'); + } + } + + return $mainDomain; + } + + /** + * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * - Domain must have proper DNS record + * + * @param Domain $domain Domain which we validate + * @param bool $isMainDomain In case of master domain, we look for different DNS configurations + * + * @return void + */ + private function validateDomain(Domain $domain, bool $isMainDomain): void { + if (empty($domain->get())) { + throw new Exception('Missing certificate domain.'); + } + + if (!$domain->isKnown() || $domain->isTest()) { + throw new Exception('Unknown public suffix for domain.'); + } + + if (!$isMainDomain) { + // TODO: Would be awesome to also support A/AAAA records here. Maybe dry run? + + // Validate if domain target is properly configured + $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); + + if (!$target->isKnown() || $target->isTest()) { + throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); + } + + // Verify domain with DNS records + $validator = new CNAME($target->get()); + if (!$validator->isValid($domain->get())) { + throw new Exception('Failed to verify domain DNS records.'); + } + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + } + } + + /** + * Reads expiry date of certificate from file and decides if renewal is required or not. + * + * @param string $domain Domain for which we check certificate file + * + * @return bool True, if certificate needs to be renewed + */ + private function isRenewRequired(string $domain): bool { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $validTo = null; + + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60*60*24*30); + if ($validTo - $expiryInAdvance > \time()) { + return false; + } + } + + return true; + } + + /** + * LetsEncrypt communication to issue certificate (using certbot CLI) + * + * @param string $domain Domain to generate certificate for + * + * @return array Named array with keys 'stdout' and 'stderr', both string + */ + private function letsEncryptAction(string $domain, string $email): array { + $staging = (App::isProduction()) ? '' : ' --dry-run'; + + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $email + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", '', $stdout, $stderr); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + return [ + 'stdout' => $stdout, + 'stderr' => $stderr + ]; + } + + /** + * Read new renew date from certificate file generated by Let's Encrypt + * + * @param string $domain Domain which certificate was generated for + * + * @return int + */ + private function getRenewDate(string $domain): int { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + $expiryInAdvance = (60*60*24*30); // 30 days + return $validTo - $expiryInAdvance; + } + + /** + * Method to take files from Let's Encrypt, and put it into Traefik. + * + * @param string $domain Domain which certificate was generated for + * + * @return void + */ + private function applyCertificateFiles(string $domain): void { + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files from certbot into our storage + if(!@\rename('/etc/letsencrypt/live/'.$domain.'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain.'/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem.'); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem.'); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem.'); + } + + if (!@\rename('/etc/letsencrypt/live/' . $domain . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem.'); + } + + $config = + "tls:" . + " certificates:" . + " - certFile: /storage/certificates/{$domain}/fullchain.pem" . + " keyFile: /storage/certificates/{$domain}/privkey.pem"; + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + } + + /** + * Method to make sure information about error is delivered to admnistrator. + * + * @param string $domain Domain that caused the error + * @param string $errorMessage Verbose error message + * @param int $attempt How many times it failed already + * + * @return void + */ + private function notifyError(string $domain, string $errorMessage, int $attempt): void { + // Log error into console + Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage); + + // Send mail to administratore mail + Resque::enqueue(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME, [ + 'from' => 'console', + 'project' => 'console', + 'name' => 'Appwrite Administrator', + 'recipient' => App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'), + 'url' => 'https://' . $domain, + 'locale' => App::getEnv('_APP_LOCALE', 'en'), + 'type' => MAIL_TYPE_CERTIFICATE, + + 'domain' => $domain, + 'error' => $errorMessage, + 'attempt' => $attempt + ]); + } + + /** + * Update all existing domain documents so they have relation to correct certificate document. + * This solved issues: + * - when adding a domain for which there is already a certificate + * - when renew creates new document? It might? + * - overall makes it more reliable + * + * @param string $certificateId ID of a new or updated certificate document + * @param string $domain Domain that is affected by new certificate + * + * @return void + */ + private function updateDomainDocuments(string $certificateId, string $domain): void { + $domains = $this->dbForConsole->find('domains', [ + new Query('domain', Query::TYPE_EQUAL, [$domain]) + ], 1000); + + foreach ($domains as $domainDocument) { + $domainDocument->setAttribute('updated', \time()); + $domainDocument->setAttribute('certificateId', $certificateId); + + $this->dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument); + $this->dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId')); + } + } } diff --git a/composer.lock b/composer.lock index c82900e1bb..4d3b65e932 100644 --- a/composer.lock +++ b/composer.lock @@ -3550,16 +3550,16 @@ }, { "name": "matthiasmullie/minify", - "version": "1.3.67", + "version": "1.3.68", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9" + "reference": "c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", - "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297", + "reference": "c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297", "shasum": "" }, "require": { @@ -3608,7 +3608,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.67" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.68" }, "funding": [ { @@ -3616,7 +3616,7 @@ "type": "github" } ], - "time": "2022-03-24T08:54:59+00:00" + "time": "2022-04-19T08:28:56+00:00" }, { "name": "matthiasmullie/path-converter", @@ -5710,16 +5710,16 @@ }, { "name": "symfony/console", - "version": "v6.0.7", + "version": "v6.0.8", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e" + "reference": "0d00aa289215353aa8746a31d101f8e60826285c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e", - "reference": "70dcf7b2ca2ea08ad6ebcc475f104a024fb5632e", + "url": "https://api.github.com/repos/symfony/console/zipball/0d00aa289215353aa8746a31d101f8e60826285c", + "reference": "0d00aa289215353aa8746a31d101f8e60826285c", "shasum": "" }, "require": { @@ -5785,7 +5785,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.0.7" + "source": "https://github.com/symfony/console/tree/v6.0.8" }, "funding": [ { @@ -5801,7 +5801,7 @@ "type": "tidelift" } ], - "time": "2022-03-31T17:18:25+00:00" + "time": "2022-04-20T15:01:42+00:00" }, { "name": "symfony/polyfill-intl-grapheme", @@ -6135,16 +6135,16 @@ }, { "name": "symfony/string", - "version": "v6.0.3", + "version": "v6.0.8", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "522144f0c4c004c80d56fa47e40e17028e2eefc2" + "reference": "ac0aa5c2282e0de624c175b68d13f2c8f2e2649d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/522144f0c4c004c80d56fa47e40e17028e2eefc2", - "reference": "522144f0c4c004c80d56fa47e40e17028e2eefc2", + "url": "https://api.github.com/repos/symfony/string/zipball/ac0aa5c2282e0de624c175b68d13f2c8f2e2649d", + "reference": "ac0aa5c2282e0de624c175b68d13f2c8f2e2649d", "shasum": "" }, "require": { @@ -6200,7 +6200,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.0.3" + "source": "https://github.com/symfony/string/tree/v6.0.8" }, "funding": [ { @@ -6216,7 +6216,7 @@ "type": "tidelift" } ], - "time": "2022-01-02T09:55:41+00:00" + "time": "2022-04-22T08:18:02+00:00" }, { "name": "textalk/websocket", From 9a3b226781c435c98f72c1a995d7d2d38bbc8931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 11 May 2022 14:09:30 +0000 Subject: [PATCH 09/10] PR review changes --- app/workers/certificates.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 0b6a45f0c3..f1df74972c 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -23,7 +23,7 @@ class CertificatesV1 extends Worker * * @var Database */ - private $dbForConsole; + private Database $dbForConsole; public function getName(): string { return "certificates"; @@ -103,7 +103,7 @@ class CertificatesV1 extends Worker } // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->letsEncryptAction($domain->get(), $email); + $letsEncryptData = $this->issueCertificate($domain->get(), $email); // Command succeeded, store all data into document // We store stderr too, because it may include warnings @@ -135,7 +135,7 @@ class CertificatesV1 extends Worker // Save all changes we made to certificate document into database $this->saveCertificateDocument($domain->get(), $certificate); - + Authorization::reset(); } } @@ -152,7 +152,7 @@ class CertificatesV1 extends Worker * * @return void */ - private function saveCertificateDocument(string $domain, Document $certificate) { + private function saveCertificateDocument(string $domain, Document $certificate): void { // Check if update or insert required $certificateDocument = $this->dbForConsole->findOne('certificates', [ new Query('domain', Query::TYPE_EQUAL, [$domain]) ]); if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { @@ -175,15 +175,15 @@ class CertificatesV1 extends Worker */ private function getMainDomain(): ?string { if (!empty(App::getEnv('_APP_DOMAIN', ''))) { - $mainDomain = App::getEnv('_APP_DOMAIN', ''); + return App::getEnv('_APP_DOMAIN', ''); } else { $domainDocument = $this->dbForConsole->findOne('domains', [], 0, ['_id'], ['ASC']); if($domainDocument) { - $mainDomain = $domainDocument->getAttribute('domain'); + return $domainDocument->getAttribute('domain'); } } - return $mainDomain; + return null; } /** @@ -262,7 +262,7 @@ class CertificatesV1 extends Worker * * @return array Named array with keys 'stdout' and 'stderr', both string */ - private function letsEncryptAction(string $domain, string $email): array { + private function issueCertificate(string $domain, string $email): array { $staging = (App::isProduction()) ? '' : ' --dry-run'; $stdout = ''; From cc8ffad56a91dbdb3ef7df68fb240c0ab1a42054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 11 May 2022 14:13:02 +0000 Subject: [PATCH 10/10] Fix lockfile --- composer.lock | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/composer.lock b/composer.lock index b50ce747c3..c2b51c52d9 100644 --- a/composer.lock +++ b/composer.lock @@ -2299,25 +2299,25 @@ }, { "name": "utopia-php/image", - "version": "0.5.3", + "version": "0.5.4", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "4a8429b62dcf56562b038d6712375f75166f0c02" + "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/4a8429b62dcf56562b038d6712375f75166f0c02", - "reference": "4a8429b62dcf56562b038d6712375f75166f0c02", + "url": "https://api.github.com/repos/utopia-php/image/zipball/ca5f436f9aa22dedaa6648f24f3687733808e336", + "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336", "shasum": "" }, "require": { "ext-imagick": "*", - "php": ">=7.4" + "php": ">=8.0" }, "require-dev": { "phpunit/phpunit": "^9.3", - "vimeo/psalm": "4.0.1" + "vimeo/psalm": "4.13.1" }, "type": "library", "autoload": { @@ -2345,9 +2345,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.5.3" + "source": "https://github.com/utopia-php/image/tree/0.5.4" }, - "time": "2021-11-02T05:47:16+00:00" + "time": "2022-05-11T12:30:41+00:00" }, { "name": "utopia-php/locale", @@ -3551,16 +3551,16 @@ }, { "name": "matthiasmullie/minify", - "version": "1.3.67", + "version": "1.3.68", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9" + "reference": "c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", - "reference": "acaee1b7ca3cd67a39d7f98673cacd7e4739a8d9", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297", + "reference": "c00fb02f71b2ef0a5f53fe18c5a8b9aa30f48297", "shasum": "" }, "require": { @@ -3609,7 +3609,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.67" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.68" }, "funding": [ { @@ -3617,7 +3617,7 @@ "type": "github" } ], - "time": "2022-03-24T08:54:59+00:00" + "time": "2022-04-19T08:28:56+00:00" }, { "name": "matthiasmullie/path-converter",