From d8dee84be1ff7cf429a7b099f6eb8c2382f96c93 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 19 Jul 2021 12:46:34 -0400 Subject: [PATCH 1/4] Obtain db connection without coroutine --- app/workers/certificates.php | 273 +++++++++++++++++------------------ 1 file changed, 132 insertions(+), 141 deletions(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 1db2a46360..8bebcca82e 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -30,178 +30,169 @@ class CertificatesV1 extends Worker { global $register; - go(function() use ($register) { - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); + $cache = new Cache(new Redis($register->get('cache'))); + $dbForConsole = new Database(new MariaDB($register->get('db')), $cache); + $dbForConsole->setNamespace('project_console_internal'); // Main DB - $cache = new Cache(new Redis($redis)); - $dbForConsole = new Database(new MariaDB($db), $cache); - $dbForConsole->setNamespace('project_console_internal'); + /** + * 1. Get new domain document - DONE + * 1.1. Validate domain is valid, public suffix is known and CNAME records are verified - DONE + * 2. Check if a certificate already exists - DONE + * 3. Check if certificate is about to expire, if not - skip it + * 3.1. Create / renew certificate + * 3.2. Update loadblancer + * 3.3. Update database (domains, change date, expiry) + * 3.4. Set retry on failure + * 3.5. Schedule to renew certificate in 60 days + */ - /** - * 1. Get new domain document - DONE - * 1.1. Validate domain is valid, public suffix is known and CNAME records are verified - DONE - * 2. Check if a certificate already exists - DONE - * 3. Check if certificate is about to expire, if not - skip it - * 3.1. Create / renew certificate - * 3.2. Update loadblancer - * 3.3. Update database (domains, change date, expiry) - * 3.4. Set retry on failure - * 3.5. Schedule to renew certificate in 60 days - */ + Authorization::disable(); - Authorization::disable(); + // Args + $document = $this->args['document']; + $domain = $this->args['domain']; - // Args - $document = $this->args['document']; - $domain = $this->args['domain']; - - // Validation Args - $validateTarget = $this->args['validateTarget'] ?? true; - $validateCNAME = $this->args['validateCNAME'] ?? true; - - // Options - $domain = new Domain((!empty($domain)) ? $domain : ''); - $expiry = 60 * 60 * 24 * 30 * 2; // 60 days - $safety = 60 * 60; // 1 hour - $renew = (\time() + $expiry); - - if(empty($domain->get())) { - throw new Exception('Missing domain'); - } - - if(!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain'); - } - - if($validateTarget) { - $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); + // Validation Args + $validateTarget = $this->args['validateTarget'] ?? true; + $validateCNAME = $this->args['validateCNAME'] ?? true; - if(!$target->isKnown() || $target->isTest()) { - throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); - } + // Options + $domain = new Domain((!empty($domain)) ? $domain : ''); + $expiry = 60 * 60 * 24 * 30 * 2; // 60 days + $safety = 60 * 60; // 1 hour + $renew = (\time() + $expiry); + + if(empty($domain->get())) { + throw new Exception('Missing domain'); + } + + if(!$domain->isKnown() || $domain->isTest()) { + throw new Exception('Unknown public suffix for domain'); + } + + if($validateTarget) { + $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); + + if(!$target->isKnown() || $target->isTest()) { + throw new Exception('Unreachable CNAME target ('.$target->get().'), please use a domain with a public suffix.'); } + } - if($validateCNAME) { - $validator = new CNAME($target->get()); // Verify Domain with DNS records - - if(!$validator->isValid($domain->get())) { - throw new Exception('Failed to verify domain DNS records'); - } + if($validateCNAME) { + $validator = new CNAME($target->get()); // Verify Domain with DNS records + + if(!$validator->isValid($domain->get())) { + throw new Exception('Failed to verify domain DNS records'); } + } - $certificate = $dbForConsole->findFirst('certificates', [ - new Query('domain', QUERY::TYPE_EQUAL, [$domain->get()]) - ], /*limit*/ 1); + $certificate = $dbForConsole->findFirst('certificates', [ + new Query('domain', QUERY::TYPE_EQUAL, [$domain->get()]) + ], /*limit*/ 1); - // $condition = ($certificate - // && $certificate instanceof Document - // && isset($certificate['issueDate']) - // && (($certificate['issueDate'] + ($expiry)) > time())) ? 'true' : 'false'; + // $condition = ($certificate + // && $certificate instanceof Document + // && isset($certificate['issueDate']) + // && (($certificate['issueDate'] + ($expiry)) > time())) ? 'true' : 'false'; - // throw new Exception('cert issued at'.date('d.m.Y H:i', $certificate['issueDate']).' | renew date is: '.date('d.m.Y H:i', ($certificate['issueDate'] + ($expiry))).' | condition is '.$condition); + // throw new Exception('cert issued at'.date('d.m.Y H:i', $certificate['issueDate']).' | renew date is: '.date('d.m.Y H:i', ($certificate['issueDate'] + ($expiry))).' | condition is '.$condition); - $certificate = (!empty($certificate) && $certificate instanceof $certificate) ? $certificate->getArrayCopy() : []; + $certificate = (!empty($certificate) && $certificate instanceof $certificate) ? $certificate->getArrayCopy() : []; - if(!empty($certificate) - && isset($certificate['issueDate']) - && (($certificate['issueDate'] + ($expiry)) > \time())) { // Check last issue time - throw new Exception('Renew isn\'t required'); + if(!empty($certificate) + && isset($certificate['issueDate']) + && (($certificate['issueDate'] + ($expiry)) > \time())) { // Check last issue time + throw new Exception('Renew isn\'t required'); + } + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); + + if(empty($email)) { + throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate'); + } + + $stdout = ''; + $stderr = ''; + + $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" + ." --email ".$email + ." -w ".APP_STORAGE_CERTIFICATES + ." -d {$domain->get()}", '', $stdout, $stderr); + + if($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: '.$stderr); + } + + $path = APP_STORAGE_CERTIFICATES.'/'.$domain->get(); + + if(!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path...'); } + } - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); + if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem: '.\json_encode($stdout)); + } - if(empty($email)) { - throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate'); - } + if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/chain.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem: '.\json_encode($stdout)); + } - $stdout = ''; - $stderr = ''; + if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/fullchain.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem: '.\json_encode($stdout)); + } - $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" - ." --email ".$email - ." -w ".APP_STORAGE_CERTIFICATES - ." -d {$domain->get()}", '', $stdout, $stderr); + if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/privkey.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem: '.\json_encode($stdout)); + } - if($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: '.$stderr); - } + $certificate = new Document(\array_merge($certificate, [ + 'domain' => $domain->get(), + 'issueDate' => \time(), + 'renewDate' => $renew, + 'attempts' => 0, + 'log' => \json_encode($stdout), + ])); - $path = APP_STORAGE_CERTIFICATES.'/'.$domain->get(); + $certificate = $dbForConsole->createDocument('certificates', $certificate); - if(!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path...'); - } - } - - if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/cert.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem: '.\json_encode($stdout)); - } + if(!$certificate) { + throw new Exception('Failed saving certificate to DB'); + } - if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/chain.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem: '.\json_encode($stdout)); - } - - if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/fullchain.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem: '.\json_encode($stdout)); - } - - if(!@\rename('/etc/letsencrypt/live/'.$domain->get().'/privkey.pem', APP_STORAGE_CERTIFICATES.'/'.$domain->get().'/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem: '.\json_encode($stdout)); - } - - $certificate = new Document(\array_merge($certificate, [ - 'domain' => $domain->get(), - 'issueDate' => \time(), - 'renewDate' => $renew, - 'attempts' => 0, - 'log' => \json_encode($stdout), + if(!empty($document)) { + $certificate = new Document(\array_merge($document, [ + 'updated' => \time(), + 'certificateId' => $certificate->getId(), ])); - $certificate = $dbForConsole->createDocument('certificates', $certificate); + $certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); if(!$certificate) { - throw new Exception('Failed saving certificate to DB'); + throw new Exception('Failed saving domain to DB'); } + } - if(!empty($document)) { - $certificate = new Document(\array_merge($document, [ - 'updated' => \time(), - 'certificateId' => $certificate->getId(), - ])); - - $certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); - - if(!$certificate) { - throw new Exception('Failed saving domain to DB'); - } - } - - $config = - "tls: - certificates: - - certFile: /storage/certificates/{$domain->get()}/fullchain.pem - keyFile: /storage/certificates/{$domain->get()}/privkey.pem"; + $config = +"tls: + certificates: + - certFile: /storage/certificates/{$domain->get()}/fullchain.pem + keyFile: /storage/certificates/{$domain->get()}/privkey.pem"; - if(!\file_put_contents(APP_STORAGE_CONFIG.'/'.$domain->get().'.yml', $config)) { - throw new Exception('Failed to save SSL configuration'); - } + if(!\file_put_contents(APP_STORAGE_CONFIG.'/'.$domain->get().'.yml', $config)) { + throw new Exception('Failed to save SSL configuration'); + } - ResqueScheduler::enqueueAt($renew + $safety, 'v1-certificates', 'CertificatesV1', [ - 'document' => [], - 'domain' => $domain->get(), - 'validateTarget' => $validateTarget, - 'validateCNAME' => $validateCNAME, - ]); // Async task rescheduale + ResqueScheduler::enqueueAt($renew + $safety, 'v1-certificates', 'CertificatesV1', [ + 'document' => [], + 'domain' => $domain->get(), + 'validateTarget' => $validateTarget, + 'validateCNAME' => $validateCNAME, + ]); // Async task rescheduale - Authorization::reset(); - - // Return db connections to pool - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - }); + Authorization::reset(); } public function shutdown(): void From f56e2fede98648dde6813b19c1dba11958f807f6 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 19 Jul 2021 12:47:02 -0400 Subject: [PATCH 2/4] Harmonize syntax for db connections --- app/workers/database.php | 31 +++++++------------------------ app/workers/deletes.php | 9 ++++----- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/app/workers/database.php b/app/workers/database.php index b024871045..55c485f304 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -2,7 +2,7 @@ use Appwrite\Resque\Worker; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Adapter\Redis; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; @@ -141,17 +141,9 @@ class DatabaseV1 extends Worker { global $register; - $dbForInternal = null; - - go(function() use ($register, $projectId, &$dbForInternal) { - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - $cache = new Cache(new RedisCache($redis)); - $dbForInternal = new Database(new MariaDB($db), $cache); - $dbForInternal->setNamespace('project_'.$projectId.'_internal'); // Main DB - - }); + $cache = new Cache(new Redis($register->get('cache'))); + $dbForInternal = new Database(new MariaDB($register->get('db')), $cache); + $dbForInternal->setNamespace('project_'.$projectId.'_internal'); // Main DB return $dbForInternal; } @@ -165,18 +157,9 @@ class DatabaseV1 extends Worker { global $register; - /** @var Database $dbForExternal */ - $dbForExternal = null; - - go(function() use ($register, $projectId, &$dbForExternal) { - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - $cache = new Cache(new RedisCache($redis)); - $dbForExternal = new Database(new MariaDB($db), $cache); - $dbForExternal->setNamespace('project_'.$projectId.'_external'); // Main DB - - }); + $cache = new Cache(new Redis($register->get('cache'))); + $dbForExternal = new Database(new MariaDB($register->get('db')), $cache); + $dbForExternal->setNamespace('project_'.$projectId.'_external'); // Main DB return $dbForExternal; } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1242d7a8ab..c855043254 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -3,14 +3,13 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Adapter\Redis; use Utopia\Database\Validator\Authorization; use Appwrite\Resque\Worker; use Utopia\Storage\Device\Local; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\CLI\Console; -use Utopia\Config\Config; use Utopia\Audit\Audit; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; @@ -360,7 +359,7 @@ class DeletesV1 extends Worker { global $register; - $cache = new Cache(new RedisCache($register->get('cache'))); + $cache = new Cache(new Redis($register->get('cache'))); $dbForInternal = new Database(new MariaDB($register->get('db')), $cache); $dbForInternal->setNamespace('project_'.$projectId.'_internal'); // Main DB @@ -375,7 +374,7 @@ class DeletesV1 extends Worker { global $register; - $cache = new Cache(new RedisCache($register->get('cache'))); + $cache = new Cache(new Redis($register->get('cache'))); $dbForExternal = new Database(new MariaDB($register->get('db')), $cache); $dbForExternal->setNamespace('project_'.$projectId.'_external'); // Main DB @@ -389,7 +388,7 @@ class DeletesV1 extends Worker { global $register; - $cache = new Cache(new RedisCache($register->get('cache'))); + $cache = new Cache(new Redis($register->get('cache'))); $dbForConsole = new Database(new MariaDB($register->get('db')), $cache); $dbForConsole->setNamespace('project_console_internal'); // Main DB From 8dab66d455a7d986847bec8edb39029ae73b4d1a Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 19 Jul 2021 14:48:03 -0400 Subject: [PATCH 3/4] Require correct dependencies in database worker --- app/workers/database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/database.php b/app/workers/database.php index 55c485f304..c9c02deeb0 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -8,7 +8,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Adapter\MariaDB; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Database V1 Worker'); Console::success(APP_NAME.' database worker v1 has started'."\n"); From 717799776a2d381c1fe597e16096cb2271fd5825 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 26 Jul 2021 12:37:29 -0400 Subject: [PATCH 4/4] Use parent class method to get consoleDB --- app/workers/certificates.php | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 8bebcca82e..1720d6adb9 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -3,13 +3,9 @@ use Appwrite\Network\Validator\CNAME; use Appwrite\Resque\Worker; use Utopia\App; -use Utopia\Cache\Adapter\Redis; -use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Validator\Authorization; use Utopia\Domains\Domain; @@ -28,11 +24,7 @@ class CertificatesV1 extends Worker public function run(): void { - global $register; - - $cache = new Cache(new Redis($register->get('cache'))); - $dbForConsole = new Database(new MariaDB($register->get('db')), $cache); - $dbForConsole->setNamespace('project_console_internal'); // Main DB + $dbForConsole = $this->getConsoleDB(); /** * 1. Get new domain document - DONE