diff --git a/.env b/.env index e849e83801..88dec63b1c 100644 --- a/.env +++ b/.env @@ -101,6 +101,7 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 +_APP_INTERVAL_DOMAIN_VERIFICATION=60 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index e146008222..ecc5112cc4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ + chmod +x /usr/local/bin/interval && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ diff --git a/bin/interval b/bin/interval new file mode 100644 index 0000000000..e4355b1dc3 --- /dev/null +++ b/bin/interval @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php interval $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3b935b84fb..805be67340 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -555,6 +555,7 @@ services: - _APP_DOMAIN_TARGET_CAA - _APP_DNS - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES - _APP_EMAIL_CERTIFICATES - _APP_REDIS_HOST - _APP_REDIS_PORT @@ -753,6 +754,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: + - mariadb - redis environment: - _APP_ENV @@ -785,6 +787,43 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES + appwrite-task-interval: + entrypoint: interval + <<: *x-logging + container_name: appwrite-task-interval + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_DOMAIN + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A + - _APP_DOMAIN_TARGET_CAA + - _APP_DNS + - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_DATABASE_SHARED_TABLES + - _APP_INTERVAL_DOMAIN_VERIFICATION + appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index ab673e9cfe..47d865ad08 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -8,6 +8,10 @@ interface Adapter { public function issueCertificate(string $certName, string $domain, ?string $domainType): ?string; + public function isInstantGeneration(string $domain, ?string $domainType): bool; + + public function getCertificateStatus(string $domain, ?string $domainType): string; + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; public function deleteCertificate(string $domain): void; diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php new file mode 100644 index 0000000000..ca15a95ed8 --- /dev/null +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -0,0 +1,10 @@ +skipRenewCheck; } + /** + * Set action for this certificate event. + * + * @param string $action + * @return self + */ + public function setAction(string $action): self + { + $this->action = $action; + return $this; + } + + /** + * Get action for this certificate event. + * + * @return string + */ + public function getAction(): string + { + return $this->action; + } + /** * Prepare the payload for the event @@ -103,7 +128,8 @@ class Certificate extends Event 'project' => $this->project, 'domain' => $this->domain, 'skipRenewCheck' => $this->skipRenewCheck, - 'validationDomain' => $this->validationDomain + 'validationDomain' => $this->validationDomain, + 'action' => $this->action ]; } } diff --git a/src/Appwrite/Platform/Action.php b/src/Appwrite/Platform/Action.php index 3db0c74d45..356209ef6f 100644 --- a/src/Appwrite/Platform/Action.php +++ b/src/Appwrite/Platform/Action.php @@ -22,9 +22,9 @@ class Action extends UtopiaAction protected mixed $logError; protected array $filters = [ - 'subQueryKeys', 'subQueryWebhooks', 'subQueryPlatforms', 'subQueryProjectVariables', 'subQueryBlocks', 'subQueryDevKeys', // Project + 'subQueryKeys', 'subQueryWebhooks', 'subQueryPlatforms', 'subQueryBlocks', 'subQueryDevKeys', // Project 'subQueryAuthenticators', 'subQuerySessions', 'subQueryTokens', 'subQueryChallenges', 'subQueryMemberships', 'subQueryTargets', 'subQueryTopicTargets',// Users - 'subQueryVariables', // Sites + 'subQueryVariables', 'subQueryProjectVariables' // Sites / Functions ]; /** diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php b/src/Appwrite/Platform/Modules/Proxy/Action.php similarity index 99% rename from src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php rename to src/Appwrite/Platform/Modules/Proxy/Action.php index 5ec5b8b8f5..c3fa535a5c 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php +++ b/src/Appwrite/Platform/Modules/Proxy/Action.php @@ -1,6 +1,6 @@ $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php index 13ee6b9cb5..ea0fb69050 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Function; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php index 4581cb3d08..4c17fdc460 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php index 7ddd501d51..f21374b49a 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Redirect; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; @@ -147,6 +147,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php index 7aff9e7e8a..26bd453eb3 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Site; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php index e09c4a6eba..7266e8d183 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Verification/Update.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules\Verification; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Modules\Proxy\Http\Rules\Action; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php index 198bf55a6f..e160b71060 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 3ada193cf7..941530d7ed 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; +use Appwrite\Platform\Tasks\Interval; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; @@ -28,6 +29,7 @@ class Tasks extends Service $this ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) + ->addAction(Interval::getName(), new Interval()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) diff --git a/src/Appwrite/Platform/Tasks/Interval.php b/src/Appwrite/Platform/Tasks/Interval.php new file mode 100644 index 0000000000..9d3d782501 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Interval.php @@ -0,0 +1,75 @@ +desc('Schedules tasks on regular intervals by publishing them to our queues') + ->inject('dbForPlatform') + ->inject('queueForCertificates') + ->callback($this->action(...)); + } + + public function action(Database $dbForPlatform, Certificate $queueForCertificates): void + { + Console::title('Interval V1'); + Console::success(APP_NAME . ' interval process v1 has started'); + + $intervalDomainVerification = (int) System::getEnv('_APP_INTERVAL_DOMAIN_VERIFICATION', '60'); // 1 minute + + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalDomainVerification) { + Console::loop(function () use ($dbForPlatform, $queueForCertificates) { + $this->verifyDomain($dbForPlatform, $queueForCertificates); + }, $intervalDomainVerification); + }); + } + + private function verifyDomain(Database $dbForPlatform, Certificate $queueForCertificates): void + { + $time = DatabaseDateTime::now(); + $fromTime = new DateTime('-3 days'); // Max 3 days old + + $rules = $dbForPlatform->find('rules', [ + Query::createdAfter(DatabaseDateTime::format($fromTime)), + Query::equal('status', [RULE_STATUS_CREATED]), // Created but not verified yet + Query::orderAsc('$updatedAt'), // Pick the ones waiting for another attempt for longest + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), // Only current region + Query::limit(30), // Reasonable pagination limit, processable within a minute + ]); + + if (\count($rules) === 0) { + Console::info("[{$time}] No rules for domain verification."); + return; // No rules to verify + } + + Console::info("[{$time}] Found " . \count($rules) . " rules for domain verification, scheduling jobs."); + + foreach ($rules as $rule) { + $queueForCertificates + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_DOMAIN_VERIFICATION) + ->trigger(); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 9c88bc4d4e..c0914c6544 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -125,33 +125,36 @@ class Maintenance extends Action Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]); + if (\count($certificates) === 0) { + Console::info("[{$time}] No certificates for renewal."); + return; + } - if (\count($certificates) > 0) { - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $appRegion = System::getEnv('_APP_REGION', 'default'); - foreach ($certificates as $certificate) { - $domain = $certificate->getAttribute('domain'); - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ + foreach ($certificates as $certificate) { + $domain = $certificate->getAttribute('domain'); + $rule = $isMd5 ? + $dbForPlatform->getDocument('rules', md5($domain)) : + $dbForPlatform->findOne('rules', [ Query::equal('domain', [$domain]), + Query::limit(1) ]); - if ($rule->isEmpty() || $rule->getAttribute('region') !== System::getEnv('_APP_REGION', 'default')) { - continue; - } - - $queueForCertificate - ->setDomain(new Document([ - 'domain' => $certificate->getAttribute('domain') - ])) - ->trigger(); + if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { + continue; } - } else { - Console::info("[{$time}] No certificates for renewal."); + + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); } } diff --git a/src/Appwrite/Platform/Tasks/SDKs.php b/src/Appwrite/Platform/Tasks/SDKs.php index 5d8cb98ae7..d6e3efa559 100644 --- a/src/Appwrite/Platform/Tasks/SDKs.php +++ b/src/Appwrite/Platform/Tasks/SDKs.php @@ -138,7 +138,7 @@ class SDKs extends Action $target = \realpath(__DIR__ . '/../../../../app') . '/sdks/git/' . $language['key'] . '/'; $readme = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/README.md'); $readme = ($readme) ? \file_get_contents($readme) : ''; - $gettingStarted = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); + $gettingStarted = $language['gettingStarted'] ?? \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); $gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : ''; $examples = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md'); $examples = ($examples) ? \file_get_contents($examples) : ''; @@ -193,8 +193,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND break; case 'php': $config = new PHP(); - $config->setComposerVendor('appwrite'); - $config->setComposerPackage('appwrite'); + $config->setComposerVendor($language['composerVendor'] ?? 'appwrite'); + $config->setComposerPackage($language['composerPackage'] ?? 'appwrite'); break; case 'nodejs': $config = new Node(); @@ -380,8 +380,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND $sdk ->setName($language['name']) ->setNamespace($language['namespace'] ?? 'appwrite') - ->setDescription("Appwrite is an open-source backend as a service server that abstract and simplify complex and repetitive development tasks behind a very simple to use REST API. Appwrite aims to help you develop your apps faster and in a more secure way. Use the {$language['name']} SDK to integrate your app with the Appwrite server to easily start interacting with all of Appwrite backend APIs and tools. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)") - ->setShortDescription('Appwrite is an open-source self-hosted backend server that abstract and simplify complex and repetitive development tasks behind a very simple REST API') + ->setDescription($language['description'] ?? "Appwrite is an open-source backend as a service server that abstracts and simplifies complex and repetitive development tasks behind a very simple to use REST API. Appwrite aims to help you develop your apps faster and in a more secure way. Use the {$language['name']} SDK to integrate your app with the Appwrite server to easily start interacting with all of Appwrite backend APIs and tools. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)") + ->setShortDescription($language['shortDescription'] ?? 'Appwrite is an open-source self-hosted backend server that abstracts and simplifies complex and repetitive development tasks behind a very simple REST API') ->setLicense($license) ->setLicenseContent($licenseContent) ->setVersion($language['version']) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 9dc6322163..5132687279 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -3,12 +3,14 @@ namespace Appwrite\Platform\Workers; use Appwrite\Certificates\Adapter as CertificatesAdapter; +use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; -use Appwrite\Network\Validator\DNS; +use Appwrite\Extend\Exception as AppwriteException; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; use Exception; @@ -22,15 +24,12 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; -use Utopia\DNS\Message\Record; +use Utopia\Database\Validator\Authorization as ValidatorAuthorization; use Utopia\Domains\Domain; use Utopia\Locale\Locale; use Utopia\Logger\Log; -use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; -use Utopia\Validator\AnyOf; -use Utopia\Validator\IP; class Certificates extends Action { @@ -42,8 +41,10 @@ class Certificates extends Action /** * @throws Exception */ - public function __construct() + public function __construct(...$params) { + parent::__construct(...$params); + $this ->desc('Certificates worker') ->inject('message') @@ -53,6 +54,7 @@ class Certificates extends Action ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') + ->inject('queueForCertificates') ->inject('log') ->inject('certificates') ->inject('plan') @@ -67,6 +69,7 @@ class Certificates extends Action * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime + * @param Certificate $queueForCertificates * @param Log $log * @param CertificatesAdapter $certificates * @return void @@ -81,6 +84,7 @@ class Certificates extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, + Certificate $queueForCertificates, Log $log, CertificatesAdapter $certificates, array $plan @@ -93,14 +97,96 @@ class Certificates extends Action $document = new Document($payload['domain'] ?? []); $domain = new Domain($document->getAttribute('domain', '')); + $domainType = $document->getAttribute('domainType'); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; $validationDomain = $payload['validationDomain'] ?? null; + $action = $payload['action'] ?? Certificate::ACTION_GENERATION; $log->addTag('domain', $domain->get()); - $domainType = $document->getAttribute('domainType'); + switch ($action) { + case Certificate::ACTION_DOMAIN_VERIFICATION: + $this->handleDomainVerificationAction($domain, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForCertificates, $log, $validationDomain); + break; - $this->execute($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + case Certificate::ACTION_GENERATION: + $this->handleCertificateGenerationAction($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + break; + + default: + throw new Exception('Invalid action: ' . $action); + } + } + + /** + * @param Domain $domain + * @param Database $dbForPlatform + * @param Event $queueForEvents + * @param Webhook $queueForWebhooks + * @param Func $queueForFunctions + * @param Realtime $queueForRealtime + * @param Certificate $queueForCertificates + * @param Log $log + * @param string|null $validationDomain + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + private function handleDomainVerificationAction( + Domain $domain, + Database $dbForPlatform, + Event $queueForEvents, + Webhook $queueForWebhooks, + Func $queueForFunctions, + Realtime $queueForRealtime, + Certificate $queueForCertificates, + Log $log, + ?string $validationDomain = null + ): void { + // Get rule + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); + + // Skip if rule is not desired state (created but not verified yet). + if ($rule->getAttribute('status', '') !== RULE_STATUS_CREATED) { + Console::warning('Domain verification for ' . $rule->getAttribute('domain', '') . ' is not needed.'); + return; + } + + Console::info('Domain verification for ' . $rule->getAttribute('domain', '') . ' started.'); + + try { + // Verify DNS records + $this->validateDomain($rule, $domain, $log, $validationDomain); + // Reset logs and status for the rule + $rule->setAttribute('logs', ''); + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); + + Console::success('Domain verification succeeded.'); + } catch (AppwriteException $err) { + Console::warning('Domain verification failed: ' . $err->getMessage()); + $rule->setAttribute('logs', $err->getMessage()); + } finally { + // Update rule and emit events + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + } + + // Issue a TLS certificate when domain is verified + if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { + $queueForCertificates + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + + Console::success('Certificate generation triggered successfully.'); + } } /** @@ -119,7 +205,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute( + private function handleCertificateGenerationAction( Domain $domain, ?string $domainType, Database $dbForPlatform, @@ -163,26 +249,42 @@ class Certificates extends Action * Note: Renewals are checked and scheduled from maintenance worker */ - // Get current certificate - $certificate = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain->get()])]); + // Get rule document for domain + // TODO: (@Meldiron) Remove after 1.7.x migration + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); - // If we don't have certificate for domain yet, let's create new document. At the end we save it + // Rule not found (or) not in the expected state + if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { + Console::warning('Certificate generation for ' . $domain->get() . ' is skipped as the associated rule is either empty or not in the expected state.'); + return; + } + + // Get associated certificate for the rule + $certificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + // If we don't have certificate for the rule yet, let's create one. if ($certificate->isEmpty()) { $certificate = new Document(); $certificate->setAttribute('domain', $domain->get()); } - $success = false; - try { $date = \date('H:i:s'); $certificate->setAttribute('logs', "\033[90m[{$date}] \033[97mCertificate generation started. \033[0m\n"); + // Persist ASAP so that logs are reset in retry flow and user can see the latest logs on Console. + $certificate = $this->upsertCertificate($rule, $certificate, $dbForPlatform); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { - $mainDomain = $validationDomain ?? $this->getMainDomain(); - $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain, $log); + $this->validateDomain($rule, $domain, $log, $validationDomain); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -191,85 +293,171 @@ class Certificates extends Action } } - // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + // Prepare unique cert name. Using this helps prevent mismatch in configuration when renewing certificates. $certName = ID::unique(); $renewDate = $certificates->issueCertificate($certName, $domain->get(), $domainType); - // Command succeeded, store all data into document - $certificate->setAttribute('logs', 'Certificate successfully generated.'); + // If certificate is generated instantly, we can mark the rule as 'verified'. + if ($certificates->isInstantGeneration($domain->get(), $domainType)) { + $rule->setAttribute('status', RULE_STATUS_VERIFIED); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); + } - // Update certificate info stored in database - $certificate->setAttribute('renewDate', $renewDate); - $certificate->setAttribute('attempts', 0); - $certificate->setAttribute('issueDate', DateTime::now()); - $success = true; + $certificate->setAttributes([ + 'attempts' => 0, // Reset attempts count + 'issueDate' => DateTime::now(), // Store current time as issue date + 'renewDate' => $renewDate, + ]); } catch (Throwable $e) { $logs = $e->getMessage(); $currentLogs = $certificate->getAttribute('logs', ''); $date = \date('H:i:s'); $errorMessage = "\033[90m[{$date}] \033[31mCertificate generation failed: \033[0m\n"; - $certificate->setAttribute('logs', $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000));// Limit to 500kb + $attempts = $certificate->getAttribute('attempts', 0) + 1; // Increase attempts count - // Increase attempts count - $attempts = $certificate->getAttribute('attempts', 0) + 1; - $certificate->setAttribute('attempts', $attempts); + // Update attributes on certificate document + $certificate->setAttributes([ + 'logs' => $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000), // Limit to 500kb + 'attempts' => $attempts, + 'renewDate' => DateTime::now(), // Store current time as renew date to ensure another attempt in next maintenance cycle. + ]); - // Store current time as renew date to ensure another attempt in next maintenance cycle. - $certificate->setAttribute('renewDate', DateTime::now()); + // Mark rule as 'unverified' + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATION_FAILED); // Send email to security email $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails, $plan); throw $e; } finally { - // All actions result in new updatedAt date + // All actions result in new 'updated' date $certificate->setAttribute('updated', DateTime::now()); + // Save certificate document to database + $this->upsertCertificate($rule, $certificate, $dbForPlatform); - // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Update rule and emit events + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } /** - * Save certificate data into database. + * Save certificate data to database. * - * @param string $domain Domain name that certificate is for + * @param Document $rule Rule associated with the domain * @param Document $certificate Certificate document that we need to save - * @param bool $success * @param Database $dbForPlatform Database connection for console - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @return void + * @return Document * @throws \Utopia\Database\Exception * @throws Authorization * @throws Conflict * @throws Structure */ - private function saveCertificateDocument( - string $domain, + private function upsertCertificate( + Document $rule, Document $certificate, - bool $success, + Database $dbForPlatform, + ): Document { + // Decide whether update (or) insert is needed + $existingCertificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + if ($existingCertificate->isEmpty()) { + $certificate->removeAttribute('$sequence'); + $certificate = $dbForPlatform->createDocument('certificates', $certificate); + } else { + $certificate = new Document(\array_merge($existingCertificate->getArrayCopy(), $certificate->getArrayCopy())); + $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); + } + + return $certificate; + } + + /** + * Update all existing domain documents so they have relation to correct certificate document. + * This solves issues: + * - when adding a domain for which there is already a certificate + * - when renew creates new document? It might? + * - overall makes it more reliable + * + * @param Document $rule Rule document that is affected by new certificate + * @param Database $dbForPlatform Database connection for console + * @param Event $queueForEvents Event publisher for events + * @param Webhook $queueForWebhooks Webhook publisher for webhooks + * @param Func $queueForFunctions Function publisher for functions + * @param Realtime $queueForRealtime Realtime publisher for realtime events + * + * @return void + */ + protected function updateRuleAndSendEvents( + Document $rule, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime ): void { - // Check if update or insert required - $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); - if (!$certificateDocument->isEmpty()) { - // Merge new data with current data - $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); - $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); - } else { - $certificate->removeAttribute('$sequence'); - $certificate = $dbForPlatform->createDocument('certificates', $certificate); + $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); + $projectId = $rule->getAttribute('projectId'); + + // Skip events for console project (triggered by auto-ssl generation for 1 click setups) + if ($projectId === 'console') { + return; } - $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + $project = $dbForPlatform->getDocument('projects', $projectId); + if ($project->isEmpty()) { + return; + } + + $ruleModel = new Rule(); + $queueForEvents + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + + /** Trigger Webhook */ + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + + /** Trigger Functions */ + $queueForFunctions + ->from($queueForEvents) + ->trigger(); + + /** Trigger Realtime Events */ + $queueForRealtime + ->setSubscribers(['console', $projectId]) + ->from($queueForEvents) + ->trigger(); + } + + /** + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) + * - Domain must have proper DNS record + * + * @param Document $rule Rule to validate + * @param Domain $domain Domain to validate + * @param Log $log Logger for adding metrics + * @param string|null $validationDomain Override for main domain check + * + * @return void + * @throws Exception + */ + private function validateDomain(Document $rule, Domain $domain, Log $log, ?string $validationDomain = null): void + { + $mainDomain = $validationDomain ?? $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + if (!$isMainDomain) { + $this->verifyRule($rule, $log); + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + } } /** @@ -288,74 +476,7 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported) - * - Domain must have proper DNS record - * - * @param Domain $domain Domain which we validate - * @param bool $isMainDomain In case of master domain, we look for different DNS configurations - * - * @return void - * @throws Exception - */ - private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void - { - if (empty($domain->get())) { - throw new Exception('Missing certificate domain.'); - } - - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain.'); - } - - if (!$isMainDomain) { - $validationStart = \microtime(true); - - $validators = []; - $targetCNAME = new Domain(System::getEnv('_APP_DOMAIN_TARGET_CNAME', '')); - if ($targetCNAME->isKnown() && !$targetCNAME->isTest()) { - $validators[] = new DNS($targetCNAME->get(), Record::TYPE_CNAME); - } - if ((new IP(IP::V4))->isValid(System::getEnv('_APP_DOMAIN_TARGET_A', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_A', ''), Record::TYPE_A); - } - if ((new IP(IP::V6))->isValid(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''), Record::TYPE_AAAA); - } - - // Validate if domain target is properly configured - if (empty($validators)) { - throw new Exception('At least one of domain targets environment variable must be configured.'); - } - - // Verify domain with DNS records - $validator = new AnyOf($validators, AnyOf::TYPE_STRING); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - throw new Exception('Failed to verify domain DNS records.'); - } - - // Ensure CAA won't block certificate issuance - if (!empty(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''))) { - $validationStart = \microtime(true); - $validator = new DNS(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''), Record::TYPE_CAA); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTimingCaa', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - $error = $validator->getDescription(); - $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); - throw new Exception('Failed to verify domain DNS records. CAA records do not allow Appwrite\'s certificate issuer.'); - } - } - } else { - // Main domain validation - // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? - } - } - - /** - * Method to make sure information about error is delivered to admnistrator. + * Method to make sure information about error is delivered to administrator. * * @param string $domain Domain that caused the error * @param string $errorMessage Verbose error message @@ -406,78 +527,4 @@ class Certificates extends Action ->setRecipient(System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))) ->trigger(); } - - /** - * Update all existing domain documents so they have relation to correct certificate document. - * This solved issues: - * - when adding a domain for which there is already a certificate - * - when renew creates new document? It might? - * - overall makes it more reliable - * - * @param string $certificateId ID of a new or updated certificate document - * @param string $domain Domain that is affected by new certificate - * @param bool $success Was certificate generation successful? - * - * @return void - */ - private function updateDomainDocuments( - string $certificateId, - string $domain, - bool $success, - Database $dbForPlatform, - Event $queueForEvents, - Webhook $queueForWebhooks, - Func $queueForFunctions, - Realtime $queueForRealtime - ): void { - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - ]); - - if (!$rule->isEmpty()) { - $rule->setAttribute('certificateId', $certificateId); - $rule->setAttribute('status', $success ? 'verified' : 'unverified'); - $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); - - $projectId = $rule->getAttribute('projectId'); - - // Skip events for console project (triggered by auto-ssl generation for 1 click setups) - if ($projectId === 'console') { - return; - } - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - return; - } - - $ruleModel = new Rule(); - $queueForEvents - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); - - /** Trigger Webhook */ - $queueForWebhooks - ->from($queueForEvents) - ->trigger(); - - /** Trigger Functions */ - $queueForFunctions - ->from($queueForEvents) - ->trigger(); - - /** Trigger Realtime Events */ - $queueForRealtime - ->from($queueForEvents) - ->setSubscribers(['console', $projectId]) - ->trigger(); - } - } } diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index d962ddc8a8..fba5154079 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -122,14 +122,16 @@ class Functions extends Action $log->addTag('type', $type); if (!empty($events)) { - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while ($sum >= $limit) { $functions = $dbForProject->find('functions', [ + Query::select(['$id', 'events']), // Skip variables subqueries + Query::contains('events', $events), Query::limit($limit), Query::offset($offset), - Query::orderAsc('name'), + Query::orderAsc('$sequence'), ]); $sum = \count($functions); @@ -147,6 +149,11 @@ class Functions extends Action continue; } + /** + * get variables subqueries cached + */ + $function = $dbForProject->getDocument('functions', $function->getId()); + Console::success('Iterating function: ' . $function->getAttribute('name')); $this->execute( diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 1ef348091a..e465f9cca2 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -192,13 +192,13 @@ class StatsResources extends Action } try { - $this->countForDatabase($dbForProject, $region); + $dbForProject->skipFilters(fn () => $this->countForDatabase($dbForProject, $region), ['subQueryAttributes', 'subQueryIndexes']); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]); } try { - $this->countForSitesAndFunctions($dbForProject, $region); + $dbForProject->skipFilters(fn () => $this->countForSitesAndFunctions($dbForProject, $region), ['subQueryVariables', 'subQueryProjectVariables']); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]); }