From 70a7deaa3807311c3925e6b7866ea9b7424b9b04 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Wed, 17 Dec 2025 20:46:54 +0530 Subject: [PATCH 01/15] Refactor certificate generation in worker --- docker-compose.yml | 1 + src/Appwrite/Certificates/Adapter.php | 2 + src/Appwrite/Certificates/LetsEncrypt.php | 5 + .../Modules/Proxy/{Http/Rules => }/Action.php | 2 +- .../Modules/Proxy/Http/Rules/API/Create.php | 2 +- .../Proxy/Http/Rules/Function/Create.php | 2 +- .../Proxy/Http/Rules/Redirect/Create.php | 2 +- .../Modules/Proxy/Http/Rules/Site/Create.php | 2 +- .../Proxy/Http/Rules/Verification/Update.php | 2 +- .../Platform/Workers/Certificates.php | 309 ++++++++---------- 10 files changed, 148 insertions(+), 181 deletions(-) rename src/Appwrite/Platform/Modules/Proxy/{Http/Rules => }/Action.php (99%) diff --git a/docker-compose.yml b/docker-compose.yml index 57007c4efc..f7e8df25e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -555,6 +555,7 @@ services: - _APP_DOMAIN_TARGET_CAA - _APP_DNS - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES - _APP_EMAIL_CERTIFICATES - _APP_REDIS_HOST - _APP_REDIS_PORT diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index ab673e9cfe..121542baa1 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -8,6 +8,8 @@ interface Adapter { public function issueCertificate(string $certName, string $domain, ?string $domainType): ?string; + public function isInstantGeneration(): bool; + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; public function deleteCertificate(string $domain): void; diff --git a/src/Appwrite/Certificates/LetsEncrypt.php b/src/Appwrite/Certificates/LetsEncrypt.php index 76638d9816..14a61203a1 100644 --- a/src/Appwrite/Certificates/LetsEncrypt.php +++ b/src/Appwrite/Certificates/LetsEncrypt.php @@ -84,6 +84,11 @@ class LetsEncrypt implements Adapter return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); } + public function isInstantGeneration(): bool + { + return true; + } + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool { $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php b/src/Appwrite/Platform/Modules/Proxy/Action.php similarity index 99% rename from src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php rename to src/Appwrite/Platform/Modules/Proxy/Action.php index 5ec5b8b8f5..c3fa535a5c 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php +++ b/src/Appwrite/Platform/Modules/Proxy/Action.php @@ -1,6 +1,6 @@ desc('Certificates worker') ->inject('message') @@ -93,13 +92,12 @@ class Certificates extends Action $document = new Document($payload['domain'] ?? []); $domain = new Domain($document->getAttribute('domain', '')); + $domainType = $document->getAttribute('domainType'); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; $validationDomain = $payload['validationDomain'] ?? null; $log->addTag('domain', $domain->get()); - $domainType = $document->getAttribute('domainType'); - $this->execute($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); } @@ -163,26 +161,43 @@ class Certificates extends Action * Note: Renewals are checked and scheduled from maintenance worker */ - // Get current certificate - $certificate = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain->get()])]); + // Get rule document for domain + // TODO: (@Meldiron) Remove after 1.7.x migration + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); - // If we don't have certificate for domain yet, let's create new document. At the end we save it + // Rule not found (or) not in the expected state + if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { + Console::warning('Certificate generation for ' . $domain . ' is skipped as the associated rule is either empty or not in the expected state.'); + } + + // Get associated certificate for the rule + $certificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + // If we don't have certificate for the rule yet, let's create one. if ($certificate->isEmpty()) { $certificate = new Document(); $certificate->setAttribute('domain', $domain->get()); } - $success = false; - try { $date = \date('H:i:s'); $certificate->setAttribute('logs', "\033[90m[{$date}] \033[97mCertificate generation started. \033[0m\n"); + // Persist ASAP so that logs are reset in retry flow and user can see the latest logs on Console. + $certificate = $this->upsertCertificate($rule, $certificate, $dbForPlatform); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { $mainDomain = $validationDomain ?? $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain, $log); + $this->validateDomain($rule, $isMainDomain, $log); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -191,85 +206,148 @@ class Certificates extends Action } } - // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + // Prepare unique cert name. Using this helps prevent mismatch in configuration when renewing certificates. $certName = ID::unique(); $renewDate = $certificates->issueCertificate($certName, $domain->get(), $domainType); - // Command succeeded, store all data into document - $certificate->setAttribute('logs', 'Certificate successfully generated.'); + // If certificate is generated instantly, we can mark the rule as 'verified'. + if ($certificates->isInstantGeneration()) { + $rule->setAttribute('status', RULE_STATUS_VERIFIED); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); + } - // Update certificate info stored in database - $certificate->setAttribute('renewDate', $renewDate); - $certificate->setAttribute('attempts', 0); - $certificate->setAttribute('issueDate', DateTime::now()); - $success = true; + $certificate->setAttributes([ + 'attempts' => 0, // Reset attempts count + 'issueDate' => DateTime::now(), // Store current time as issue date + 'renewDate' => $renewDate, + ]); } catch (Throwable $e) { $logs = $e->getMessage(); $currentLogs = $certificate->getAttribute('logs', ''); $date = \date('H:i:s'); $errorMessage = "\033[90m[{$date}] \033[31mCertificate generation failed: \033[0m\n"; - $certificate->setAttribute('logs', $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000));// Limit to 500kb + $attempts = $certificate->getAttribute('attempts', 0) + 1; // // Increase attempts count - // Increase attempts count - $attempts = $certificate->getAttribute('attempts', 0) + 1; - $certificate->setAttribute('attempts', $attempts); + // Update attributes on certificate document + $certificate->setAttributes([ + 'logs' => $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000), // Limit to 500kb + 'attempts' => $attempts, + 'renewDate' => DateTime::now(), // Store current time as renew date to ensure another attempt in next maintenance cycle. + ]); - // Store current time as renew date to ensure another attempt in next maintenance cycle. - $certificate->setAttribute('renewDate', DateTime::now()); + // Mark rule as 'unverified' + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATION_FAILED); // Send email to security email $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails, $plan); throw $e; } finally { - // All actions result in new updatedAt date + // All actions result in new 'updated' date $certificate->setAttribute('updated', DateTime::now()); + // Save certificate document to database + $this->upsertCertificate($rule, $certificate, $dbForPlatform); - // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Update rule and emit events + $this->updateDomainDocuments($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } /** - * Save certificate data into database. + * Save certificate data to database. * - * @param string $domain Domain name that certificate is for + * @param Document $rule Rule associated with the domain * @param Document $certificate Certificate document that we need to save - * @param bool $success * @param Database $dbForPlatform Database connection for console - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @return void + * @return Document * @throws \Utopia\Database\Exception * @throws Authorization * @throws Conflict * @throws Structure */ - private function saveCertificateDocument( - string $domain, + private function upsertCertificate( + Document $rule, Document $certificate, - bool $success, + Database $dbForPlatform, + ): Document { + // Decide whether update (or) insert is needed + $existingCertificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + if ($existingCertificate->isEmpty()) { + $certificate->removeAttribute('$sequence'); + $certificate = $dbForPlatform->createDocument('certificates', $certificate); + } else { + $certificate = new Document(\array_merge($existingCertificate->getArrayCopy(), $certificate->getArrayCopy())); + $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); + } + + return $certificate; + } + + /** + * Update all existing domain documents so they have relation to correct certificate document. + * This solved issues: + * - when adding a domain for which there is already a certificate + * - when renew creates new document? It might? + * - overall makes it more reliable + * + * @param Document $rule Rule document that is affected by new certificate + * @param Database $dbForPlatform Database connection for console + * @param Event $queueForEvents Event publisher for events + * @param Webhook $queueForWebhooks Webhook publisher for webhooks + * @param Func $queueForFunctions Function publisher for functions + * @param Realtime $queueForRealtime Realtime publisher for realtime events + * + * @return void + */ + private function updateDomainDocuments( + Document $rule, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime ): void { - // Check if update or insert required - $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); - if (!$certificateDocument->isEmpty()) { - // Merge new data with current data - $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); - $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); - } else { - $certificate->removeAttribute('$sequence'); - $certificate = $dbForPlatform->createDocument('certificates', $certificate); + $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); + + $projectId = $rule->getAttribute('projectId'); + + // Skip events for console project (triggered by auto-ssl generation for 1 click setups) + if ($projectId === 'console') { + return; } - $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + $project = $dbForPlatform->getDocument('projects', $projectId); + + if ($project->isEmpty()) { + return; + } + + $ruleModel = new Rule(); + $queueForEvents + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + + /** Trigger Webhook */ + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + + /** Trigger Functions */ + $queueForFunctions + ->from($queueForEvents) + ->trigger(); + + /** Trigger Realtime Events */ + $queueForRealtime + ->setSubscribers(['console', $projectId]) + ->from($queueForEvents) + ->trigger(); } /** @@ -292,62 +370,17 @@ class Certificates extends Action * - Domain needs to be public and valid (prevents NFT domains that are not supported) * - Domain must have proper DNS record * - * @param Domain $domain Domain which we validate + * @param Document $rule Rule to validate * @param bool $isMainDomain In case of master domain, we look for different DNS configurations + * @param Log $log Logger for adding metrics * * @return void * @throws Exception */ - private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void + private function validateDomain(Document $rule, bool $isMainDomain, Log $log): void { - if (empty($domain->get())) { - throw new Exception('Missing certificate domain.'); - } - - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain.'); - } - if (!$isMainDomain) { - $validationStart = \microtime(true); - - $validators = []; - $targetCNAME = new Domain(System::getEnv('_APP_DOMAIN_TARGET_CNAME', '')); - if ($targetCNAME->isKnown() && !$targetCNAME->isTest()) { - $validators[] = new DNS($targetCNAME->get(), Record::TYPE_CNAME); - } - if ((new IP(IP::V4))->isValid(System::getEnv('_APP_DOMAIN_TARGET_A', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_A', ''), Record::TYPE_A); - } - if ((new IP(IP::V6))->isValid(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''), Record::TYPE_AAAA); - } - - // Validate if domain target is properly configured - if (empty($validators)) { - throw new Exception('At least one of domain targets environment variable must be configured.'); - } - - // Verify domain with DNS records - $validator = new AnyOf($validators, AnyOf::TYPE_STRING); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - throw new Exception('Failed to verify domain DNS records.'); - } - - // Ensure CAA won't block certificate issuance - if (!empty(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''))) { - $validationStart = \microtime(true); - $validator = new DNS(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''), Record::TYPE_CAA); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTimingCaa', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - $error = $validator->getDescription(); - $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); - throw new Exception('Failed to verify domain DNS records. CAA records do not allow Appwrite\'s certificate issuer.'); - } - } + $this->verifyRule($rule, $log); } else { // Main domain validation // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? @@ -406,78 +439,4 @@ class Certificates extends Action ->setRecipient(System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))) ->trigger(); } - - /** - * Update all existing domain documents so they have relation to correct certificate document. - * This solved issues: - * - when adding a domain for which there is already a certificate - * - when renew creates new document? It might? - * - overall makes it more reliable - * - * @param string $certificateId ID of a new or updated certificate document - * @param string $domain Domain that is affected by new certificate - * @param bool $success Was certificate generation successful? - * - * @return void - */ - private function updateDomainDocuments( - string $certificateId, - string $domain, - bool $success, - Database $dbForPlatform, - Event $queueForEvents, - Webhook $queueForWebhooks, - Func $queueForFunctions, - Realtime $queueForRealtime - ): void { - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - ]); - - if (!$rule->isEmpty()) { - $rule->setAttribute('certificateId', $certificateId); - $rule->setAttribute('status', $success ? 'verified' : 'unverified'); - $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); - - $projectId = $rule->getAttribute('projectId'); - - // Skip events for console project (triggered by auto-ssl generation for 1 click setups) - if ($projectId === 'console') { - return; - } - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - return; - } - - $ruleModel = new Rule(); - $queueForEvents - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); - - /** Trigger Webhook */ - $queueForWebhooks - ->from($queueForEvents) - ->trigger(); - - /** Trigger Functions */ - $queueForFunctions - ->from($queueForEvents) - ->trigger(); - - /** Trigger Realtime Events */ - $queueForRealtime - ->from($queueForEvents) - ->setSubscribers(['console', $projectId]) - ->trigger(); - } - } } From 0e27088e2aa3457fca73909798055df2574ee4f1 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Wed, 17 Dec 2025 20:54:18 +0530 Subject: [PATCH 02/15] change function signature --- src/Appwrite/Certificates/Adapter.php | 2 +- src/Appwrite/Certificates/LetsEncrypt.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index 121542baa1..770d2bb71d 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -8,7 +8,7 @@ interface Adapter { public function issueCertificate(string $certName, string $domain, ?string $domainType): ?string; - public function isInstantGeneration(): bool; + public function isInstantGeneration(string $domain, ?string $domainType): bool; public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; diff --git a/src/Appwrite/Certificates/LetsEncrypt.php b/src/Appwrite/Certificates/LetsEncrypt.php index 14a61203a1..7e71080a3c 100644 --- a/src/Appwrite/Certificates/LetsEncrypt.php +++ b/src/Appwrite/Certificates/LetsEncrypt.php @@ -84,7 +84,7 @@ class LetsEncrypt implements Adapter return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); } - public function isInstantGeneration(): bool + public function isInstantGeneration(string $domain, ?string $domainType): bool { return true; } From db0dbeb27b0d310d3ce1670b32b498920b2e2a88 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Wed, 17 Dec 2025 21:04:15 +0530 Subject: [PATCH 03/15] simplify --- .../Platform/Workers/Certificates.php | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 5f839f3850..68572961aa 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -195,9 +195,7 @@ class Certificates extends Action // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { - $mainDomain = $validationDomain ?? $this->getMainDomain(); - $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($rule, $isMainDomain, $log); + $this->validateDomain($rule, $domain, $validationDomain, $log); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -211,7 +209,7 @@ class Certificates extends Action $renewDate = $certificates->issueCertificate($certName, $domain->get(), $domainType); // If certificate is generated instantly, we can mark the rule as 'verified'. - if ($certificates->isInstantGeneration()) { + if ($certificates->isInstantGeneration($domain->get(), $domainType)) { $rule->setAttribute('status', RULE_STATUS_VERIFIED); $certificate->setAttribute('logs', 'Certificate successfully generated.'); } @@ -350,6 +348,31 @@ class Certificates extends Action ->trigger(); } + /** + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) + * - Domain must have proper DNS record + * + * @param Document $rule Rule to validate + * @param Domain $domain Domain to validate + * @param string|null $validationDomain Override for main domain check + * @param Log $log Logger for adding metrics + * + * @return void + * @throws Exception + */ + private function validateDomain(Document $rule, Domain $domain, ?string $validationDomain = null, Log $log): void + { + $mainDomain = $validationDomain ?? $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + if (!$isMainDomain) { + $this->verifyRule($rule, $log); + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + } + } + /** * Get main domain. Needed as we do different checks for main and non-main domains. * @@ -366,29 +389,7 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported) - * - Domain must have proper DNS record - * - * @param Document $rule Rule to validate - * @param bool $isMainDomain In case of master domain, we look for different DNS configurations - * @param Log $log Logger for adding metrics - * - * @return void - * @throws Exception - */ - private function validateDomain(Document $rule, bool $isMainDomain, Log $log): void - { - if (!$isMainDomain) { - $this->verifyRule($rule, $log); - } else { - // Main domain validation - // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? - } - } - - /** - * Method to make sure information about error is delivered to admnistrator. + * Method to make sure information about error is delivered to administrator. * * @param string $domain Domain that caused the error * @param string $errorMessage Verbose error message From 88bd35ce9881df475ceeb36cdba18af4e5132499 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 12:40:24 +0530 Subject: [PATCH 04/15] periodic task for rule verification & certificate generation --- .env | 2 + Dockerfile | 1 + bin/rules | 3 + docker-compose.yml | 37 +++++ src/Appwrite/Event/Certificate.php | 25 ++++ .../Modules/Proxy/Http/Rules/API/Create.php | 1 + .../Proxy/Http/Rules/Function/Create.php | 1 + .../Platform/Modules/Proxy/Http/Rules/Get.php | 1 + .../Proxy/Http/Rules/Redirect/Create.php | 1 + .../Modules/Proxy/Http/Rules/Site/Create.php | 1 + .../Modules/Proxy/Http/Rules/XList.php | 1 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/Maintenance.php | 42 ------ src/Appwrite/Platform/Tasks/Rules.php | 126 ++++++++++++++++++ .../Platform/Workers/Certificates.php | 97 +++++++++++++- 15 files changed, 292 insertions(+), 49 deletions(-) create mode 100644 bin/rules create mode 100644 src/Appwrite/Platform/Tasks/Rules.php diff --git a/.env b/.env index 64fc7ef10f..d211f0ae6e 100644 --- a/.env +++ b/.env @@ -101,6 +101,8 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 +_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL=60 +_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index e146008222..b65b73469c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -58,6 +58,7 @@ RUN mkdir -p /storage/uploads && \ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/rules && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ diff --git a/bin/rules b/bin/rules new file mode 100644 index 0000000000..77e195eb61 --- /dev/null +++ b/bin/rules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php rules $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f7e8df25e6..99b42561b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -786,6 +786,43 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES + appwrite-task-rules: + entrypoint: rules + <<: *x-logging + container_name: appwrite-task-rules + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_DOMAIN + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A + - _APP_DOMAIN_TARGET_CAA + - _APP_DNS + - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_DATABASE_SHARED_TABLES + - _APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL + - _APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL + appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index 00875c7a4a..7e60d15180 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -7,7 +7,10 @@ use Utopia\Queue\Publisher; class Certificate extends Event { + public const string ACTION_DOMAIN_VERIFICATION = 'verification'; + public const string ACTION_GENERATION = 'generation'; protected bool $skipRenewCheck = false; + protected string $action = self::ACTION_GENERATION; protected ?Document $domain = null; protected ?string $validationDomain = null; @@ -90,6 +93,28 @@ class Certificate extends Event return $this->skipRenewCheck; } + /** + * Set action for this certificate event. + * + * @param string $action + * @return self + */ + public function setAction(string $action): self + { + $this->action = $action; + return $this; + } + + /** + * Get action for this certificate event. + * + * @return string + */ + public function getAction(): string + { + return $this->action; + } + /** * Prepare the payload for the event diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php index ea2f34f8fd..95ea8dd8cf 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php @@ -125,6 +125,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php index 0009a2eb57..ea0fb69050 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php index 4581cb3d08..4c17fdc460 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php index 493c827c46..f21374b49a 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php @@ -147,6 +147,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php index 9ec79139af..26bd453eb3 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php index 198bf55a6f..e160b71060 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 3ada193cf7..3b0ba7d5ea 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -7,6 +7,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; +use Appwrite\Platform\Tasks\Rules; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; @@ -29,6 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) + ->addAction(Rules::getName(), new Rules()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 9c88bc4d4e..66d3a3d9de 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -92,7 +92,6 @@ class Maintenance extends Action ->trigger(); $this->notifyDeleteConnections($queueForDeletes); - $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); $this->notifyDeleteCSVExports($queueForDeletes); @@ -114,47 +113,6 @@ class Maintenance extends Action ->trigger(); } - private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void - { - $time = DatabaseDateTime::now(); - - $certificates = $dbForPlatform->find('certificates', [ - Query::lessThan('attempts', 5), // Maximum 5 attempts - Query::isNotNull('renewDate'), - Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) - Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) - ]); - - - if (\count($certificates) > 0) { - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - - foreach ($certificates as $certificate) { - $domain = $certificate->getAttribute('domain'); - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - ]); - - if ($rule->isEmpty() || $rule->getAttribute('region') !== System::getEnv('_APP_REGION', 'default')) { - continue; - } - - $queueForCertificate - ->setDomain(new Document([ - 'domain' => $certificate->getAttribute('domain') - ])) - ->trigger(); - } - } else { - Console::info("[{$time}] No certificates for renewal."); - } - } - private function notifyDeleteCache($interval, Delete $queueForDeletes): void { $queueForDeletes diff --git a/src/Appwrite/Platform/Tasks/Rules.php b/src/Appwrite/Platform/Tasks/Rules.php new file mode 100644 index 0000000000..1398459a88 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Rules.php @@ -0,0 +1,126 @@ +desc('Schedules periodic tasks for rule verification and certificate renewal') + ->inject('dbForPlatform') + ->inject('queueForCertificates') + ->callback($this->action(...)); + } + + public function action(Database $dbForPlatform, Certificate $queueForCertificates): void + { + Console::title('Interval V1'); + Console::success(APP_NAME . ' interval process v1 has started'); + + $intervalRuleVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL', '60'); // 1 minute + $intervalCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day + + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleVerification) { + Console::loop(function () use ($dbForPlatform, $queueForCertificates) { + $this->checkRuleVerification($dbForPlatform, $queueForCertificates); + }, $intervalRuleVerification); + }); + + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { + Console::loop(function () use ($dbForPlatform, $queueForCertificates) { + $this->renewCertificates($dbForPlatform, $queueForCertificates); + }, $intervalCertificateRenewal); + }); + } + + private function checkRuleVerification(Database $dbForPlatform, Certificate $queueForCertificate): void + { + $time = DatabaseDateTime::now(); + $fromTime = new DateTime('-3 days'); // Max 3 days old + + $rules = $dbForPlatform->find('rules', [ + Query::createdAfter(DatabaseDateTime::format($fromTime)), + Query::equal('status', [RULE_STATUS_CREATED]), // Created but not verified yet + Query::orderAsc('$updatedAt'), // Pick the ones waiting for another attempt for longest + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), // Only current region + Query::limit(30), // Reasonable pagination limit, processable within a minute + ]); + + if (\count($rules) === 0) { + Console::info("[{$time}] No rules for verification."); + return; // No rules to verify + } + + Console::info("[{$time}] Found " . \count($rules) . " rules for verification, scheduling jobs."); + + foreach ($rules as $rule) { + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_DOMAIN_VERIFICATION) + ->trigger(); + } + } + + private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void + { + $time = DatabaseDateTime::now(); + + $certificates = $dbForPlatform->find('certificates', [ + Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), + Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) + Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) + ]); + + if (\count($certificates) === 0) { + Console::info("[{$time}] No certificates for renewal."); + return; + } + + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $appRegion = System::getEnv('_APP_REGION', 'default'); + + foreach ($certificates as $certificate) { + $domain = $certificate->getAttribute('domain'); + $rule = $isMd5 ? + $dbForPlatform->getDocument('rules', md5($domain)) : + $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain]), + Query::limit(1) + ]); + + if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { + continue; + } + + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + } + } +} diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 68572961aa..a78918471a 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -3,11 +3,13 @@ namespace Appwrite\Platform\Workers; use Appwrite\Certificates\Adapter as CertificatesAdapter; +use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; +use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; @@ -20,9 +22,9 @@ use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Structure; -use Utopia\Database\Validator\Authorization as ValidatorAuthorization; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization as ValidatorAuthorization; use Utopia\Domains\Domain; use Utopia\Locale\Locale; use Utopia\Logger\Log; @@ -52,6 +54,7 @@ class Certificates extends Action ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') + ->inject('queueForCertificates') ->inject('log') ->inject('certificates') ->inject('plan') @@ -66,6 +69,7 @@ class Certificates extends Action * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime + * @param Certificate $queueForCertificates * @param Log $log * @param CertificatesAdapter $certificates * @return void @@ -80,6 +84,7 @@ class Certificates extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, + Certificate $queueForCertificates, Log $log, CertificatesAdapter $certificates, array $plan @@ -95,10 +100,88 @@ class Certificates extends Action $domainType = $document->getAttribute('domainType'); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; $validationDomain = $payload['validationDomain'] ?? null; + $action = $payload['action'] ?? Certificate::ACTION_GENERATION; $log->addTag('domain', $domain->get()); - $this->execute($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + switch ($action) { + case Certificate::ACTION_DOMAIN_VERIFICATION: + $this->handleDomainVerificationAction($domain, $dbForPlatform, $log, $queueForCertificates, $validationDomain); + break; + + case Certificate::ACTION_GENERATION: + $this->handleCertificateGenerationAction($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + break; + + default: + throw new Exception('Invalid action: ' . $action); + } + + + } + + /** + * @param Domain $domain + * @param Database $dbForPlatform + * @param Log $log + * @param Certificate $queueForCertificates + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + private function handleDomainVerificationAction( + Domain $domain, + Database $dbForPlatform, + Log $log, + Certificate $queueForCertificates, + ?string $validationDomain = null, + ): void { + // Get rule + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); + + // Skip if rule is not desired state (created but not verified yet). + if ($rule->getAttribute('status', '') !== RULE_STATUS_CREATED) { + Console::warning('Domain verification for ' . $rule->getAttribute('domain', '') . ' is not needed.'); + return; + } + + Console::info('Domain verification for ' . $rule->getAttribute('domain', '') . ' started.'); + + $updates = new Document(); + try { + // Verify DNS records + $this->validateDomain($rule, $domain, $log, $validationDomain); + // Reset logs and status for the rule + $updates + ->setAttribute('logs', '') + ->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); + + Console::success('Domain verification succeeded.'); + } catch (AppwriteException $err) { + Console::warning('Domain verification failed: ' . $err->getMessage()); + $updates->setAttribute('logs', $err->getMessage()); + } + + echo "updating rule with updates: " . \var_dump($updates); + $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $updates); + + // Issue a TLS certificate when domain is verified + if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { + $queueForCertificates + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + + Console::success('Certificate generation triggered successfully.'); + } } /** @@ -117,7 +200,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute( + private function handleCertificateGenerationAction( Domain $domain, ?string $domainType, Database $dbForPlatform, @@ -172,7 +255,7 @@ class Certificates extends Action // Rule not found (or) not in the expected state if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { - Console::warning('Certificate generation for ' . $domain . ' is skipped as the associated rule is either empty or not in the expected state.'); + Console::warning('Certificate generation for ' . $domain->get() . ' is skipped as the associated rule is either empty or not in the expected state.'); } // Get associated certificate for the rule @@ -195,7 +278,7 @@ class Certificates extends Action // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { - $this->validateDomain($rule, $domain, $validationDomain, $log); + $this->validateDomain($rule, $domain, $log, $validationDomain); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -355,13 +438,13 @@ class Certificates extends Action * * @param Document $rule Rule to validate * @param Domain $domain Domain to validate - * @param string|null $validationDomain Override for main domain check * @param Log $log Logger for adding metrics + * @param string|null $validationDomain Override for main domain check * * @return void * @throws Exception */ - private function validateDomain(Document $rule, Domain $domain, ?string $validationDomain = null, Log $log): void + private function validateDomain(Document $rule, Domain $domain, Log $log, ?string $validationDomain = null): void { $mainDomain = $validationDomain ?? $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; From 6f1d2d094a14e4972430e727cb68dd45b8c54ad5 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 13:00:49 +0530 Subject: [PATCH 05/15] rename task --- .env | 4 ++-- Dockerfile | 2 +- bin/maintenance-rules | 3 +++ bin/rules | 3 --- docker-compose.yml | 10 +++++----- src/Appwrite/Platform/Services/Tasks.php | 4 ++-- .../Tasks/{Rules.php => MaintenanceRules.php} | 16 ++++++++-------- 7 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 bin/maintenance-rules delete mode 100644 bin/rules rename src/Appwrite/Platform/Tasks/{Rules.php => MaintenanceRules.php} (89%) diff --git a/.env b/.env index d211f0ae6e..be0e5df718 100644 --- a/.env +++ b/.env @@ -101,8 +101,8 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL=60 -_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL=86400 +_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL=60 +_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index b65b73469c..71baa9e1c6 100755 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ RUN mkdir -p /storage/uploads && \ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/maintenance && \ - chmod +x /usr/local/bin/rules && \ + chmod +x /usr/local/bin/maintenance-rules && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ diff --git a/bin/maintenance-rules b/bin/maintenance-rules new file mode 100644 index 0000000000..666e517ca0 --- /dev/null +++ b/bin/maintenance-rules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php maintenance-rules $@ \ No newline at end of file diff --git a/bin/rules b/bin/rules deleted file mode 100644 index 77e195eb61..0000000000 --- a/bin/rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php rules $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 99b42561b3..5490d19b68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -786,10 +786,10 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES - appwrite-task-rules: - entrypoint: rules + appwrite-task-maintenance-rules: + entrypoint: maintenance-rules <<: *x-logging - container_name: appwrite-task-rules + container_name: appwrite-task-maintenance-rules image: appwrite-dev networks: - appwrite @@ -820,8 +820,8 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL - - _APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL + - _APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL + - _APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL appwrite-task-stats-resources: container_name: appwrite-task-stats-resources diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 3b0ba7d5ea..f68e3922ac 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -7,7 +7,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; -use Appwrite\Platform\Tasks\Rules; +use Appwrite\Platform\Tasks\MaintenanceRules; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; @@ -30,7 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(Rules::getName(), new Rules()) + ->addAction(MaintenanceRules::getName(), new MaintenanceRules()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/Rules.php b/src/Appwrite/Platform/Tasks/MaintenanceRules.php similarity index 89% rename from src/Appwrite/Platform/Tasks/Rules.php rename to src/Appwrite/Platform/Tasks/MaintenanceRules.php index 1398459a88..03e08bda0a 100644 --- a/src/Appwrite/Platform/Tasks/Rules.php +++ b/src/Appwrite/Platform/Tasks/MaintenanceRules.php @@ -12,11 +12,11 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\System\System; -class Rules extends Action +class MaintenanceRules extends Action { public static function getName(): string { - return 'rules'; + return 'maintenance-rules'; } public function __construct() @@ -33,19 +33,19 @@ class Rules extends Action Console::title('Interval V1'); Console::success(APP_NAME . ' interval process v1 has started'); - $intervalRuleVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL', '60'); // 1 minute - $intervalCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day + $intervalRuleDomainVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL', '60'); // 1 minute + $intervalRuleCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleVerification) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->checkRuleVerification($dbForPlatform, $queueForCertificates); - }, $intervalRuleVerification); + }, $intervalRuleDomainVerification); }); - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleCertificateRenewal) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->renewCertificates($dbForPlatform, $queueForCertificates); - }, $intervalCertificateRenewal); + }, $intervalRuleCertificateRenewal); }); } From 62b391ef680c6a183e1484d6425f103b46470f1e Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 13:46:56 +0530 Subject: [PATCH 06/15] certificate status exception --- src/Appwrite/Certificates/Adapter.php | 2 ++ .../Certificates/Exception/CertificateStatus.php | 10 ++++++++++ src/Appwrite/Certificates/LetsEncrypt.php | 6 ++++++ src/Appwrite/Platform/Tasks/MaintenanceRules.php | 4 ++-- 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 src/Appwrite/Certificates/Exception/CertificateStatus.php diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index 770d2bb71d..47d865ad08 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -10,6 +10,8 @@ interface Adapter public function isInstantGeneration(string $domain, ?string $domainType): bool; + public function getCertificateStatus(string $domain, ?string $domainType): string; + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; public function deleteCertificate(string $domain): void; diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php new file mode 100644 index 0000000000..652c1c4253 --- /dev/null +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -0,0 +1,10 @@ + Date: Thu, 18 Dec 2025 15:14:30 +0530 Subject: [PATCH 07/15] add action to payload --- docker-compose.yml | 2 ++ src/Appwrite/Event/Certificate.php | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5490d19b68..30b0737543 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -754,6 +754,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: + - mariadb - redis environment: - _APP_ENV @@ -797,6 +798,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: + - mariadb - redis environment: - _APP_ENV diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index 7e60d15180..4ad12094c2 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -127,7 +127,8 @@ class Certificate extends Event 'project' => $this->project, 'domain' => $this->domain, 'skipRenewCheck' => $this->skipRenewCheck, - 'validationDomain' => $this->validationDomain + 'validationDomain' => $this->validationDomain, + 'action' => $this->action ]; } } From aa4ecdf13898eafa3111e8301f0b88b874570caf Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 16:07:15 +0530 Subject: [PATCH 08/15] emit events --- .../Platform/Workers/Certificates.php | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index a78918471a..5c3fe9aede 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -106,7 +106,7 @@ class Certificates extends Action switch ($action) { case Certificate::ACTION_DOMAIN_VERIFICATION: - $this->handleDomainVerificationAction($domain, $dbForPlatform, $log, $queueForCertificates, $validationDomain); + $this->handleDomainVerificationAction($domain, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForCertificates, $log, $validationDomain); break; case Certificate::ACTION_GENERATION: @@ -123,8 +123,13 @@ class Certificates extends Action /** * @param Domain $domain * @param Database $dbForPlatform - * @param Log $log + * @param Event $queueForEvents + * @param Webhook $queueForWebhooks + * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param Certificate $queueForCertificates + * @param Log $log + * @param string|null $validationDomain * @return void * @throws Throwable * @throws \Utopia\Database\Exception @@ -132,8 +137,12 @@ class Certificates extends Action private function handleDomainVerificationAction( Domain $domain, Database $dbForPlatform, - Log $log, + Event $queueForEvents, + Webhook $queueForWebhooks, + Func $queueForFunctions, + Realtime $queueForRealtime, Certificate $queueForCertificates, + Log $log, ?string $validationDomain = null, ): void { // Get rule @@ -152,23 +161,22 @@ class Certificates extends Action Console::info('Domain verification for ' . $rule->getAttribute('domain', '') . ' started.'); - $updates = new Document(); try { // Verify DNS records $this->validateDomain($rule, $domain, $log, $validationDomain); // Reset logs and status for the rule - $updates - ->setAttribute('logs', '') - ->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); + $rule->setAttribute('logs', ''); + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); Console::success('Domain verification succeeded.'); } catch (AppwriteException $err) { Console::warning('Domain verification failed: ' . $err->getMessage()); - $updates->setAttribute('logs', $err->getMessage()); + $rule->setAttribute('logs', $err->getMessage()); + } finally { + // Update rule and emit events + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } - echo "updating rule with updates: " . \var_dump($updates); - $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $updates); // Issue a TLS certificate when domain is verified if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { @@ -333,7 +341,7 @@ class Certificates extends Action // Ensure certificate is associated with the rule $rule->setAttribute('certificateId', $certificate->getId()); // Update rule and emit events - $this->updateDomainDocuments($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } @@ -384,7 +392,7 @@ class Certificates extends Action * * @return void */ - private function updateDomainDocuments( + protected function updateRuleAndSendEvents( Document $rule, Database $dbForPlatform, Event $queueForEvents, From e78887001afb86af53311f31834bf227f2de702a Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 16:18:23 +0530 Subject: [PATCH 09/15] spacing --- src/Appwrite/Platform/Workers/Certificates.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 5c3fe9aede..6371f6c313 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -401,7 +401,6 @@ class Certificates extends Action Realtime $queueForRealtime ): void { $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); - $projectId = $rule->getAttribute('projectId'); // Skip events for console project (triggered by auto-ssl generation for 1 click setups) @@ -410,7 +409,6 @@ class Certificates extends Action } $project = $dbForPlatform->getDocument('projects', $projectId); - if ($project->isEmpty()) { return; } From 16fb25ce5a9e74961e7b3a0ef40ebb47d7b34f3b Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 17:10:02 +0530 Subject: [PATCH 10/15] tiny --- src/Appwrite/Platform/Tasks/MaintenanceRules.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/MaintenanceRules.php b/src/Appwrite/Platform/Tasks/MaintenanceRules.php index c83207063a..cbcd538d8c 100644 --- a/src/Appwrite/Platform/Tasks/MaintenanceRules.php +++ b/src/Appwrite/Platform/Tasks/MaintenanceRules.php @@ -38,7 +38,7 @@ class MaintenanceRules extends Action \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { - $this->checkRuleVerification($dbForPlatform, $queueForCertificates); + $this->verifyDomain($dbForPlatform, $queueForCertificates); }, $intervalRuleDomainVerification); }); @@ -49,7 +49,7 @@ class MaintenanceRules extends Action }); } - private function checkRuleVerification(Database $dbForPlatform, Certificate $queueForCertificate): void + private function verifyDomain(Database $dbForPlatform, Certificate $queueForCertificate): void { $time = DatabaseDateTime::now(); $fromTime = new DateTime('-3 days'); // Max 3 days old @@ -63,11 +63,11 @@ class MaintenanceRules extends Action ]); if (\count($rules) === 0) { - Console::info("[{$time}] No rules for verification."); + Console::info("[{$time}] No rules for domain verification."); return; // No rules to verify } - Console::info("[{$time}] Found " . \count($rules) . " rules for verification, scheduling jobs."); + Console::info("[{$time}] Found " . \count($rules) . " rules for domain verification, scheduling jobs."); foreach ($rules as $rule) { $queueForCertificate From cf90653deb7d44ae7f42eff80725e083e3f07ff9 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 17:30:08 +0530 Subject: [PATCH 11/15] lint --- src/Appwrite/Certificates/Exception/CertificateStatus.php | 2 +- src/Appwrite/Platform/Services/Tasks.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php index 652c1c4253..3d94109d0e 100644 --- a/src/Appwrite/Certificates/Exception/CertificateStatus.php +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -7,4 +7,4 @@ use Exception; // Exception thrown during certificate status retrieval class CertificateStatus extends Exception { -} \ No newline at end of file +} diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index f68e3922ac..f585557690 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -5,9 +5,9 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; +use Appwrite\Platform\Tasks\MaintenanceRules; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; -use Appwrite\Platform\Tasks\MaintenanceRules; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; From 47ea2893c3bb469edf3048e95004f7eda3ff149e Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Tue, 23 Dec 2025 16:32:43 +0530 Subject: [PATCH 12/15] rename task to Interval --- .env | 4 ++-- Dockerfile | 2 +- bin/interval | 3 +++ bin/maintenance-rules | 3 --- docker-compose.yml | 10 ++++----- src/Appwrite/Platform/Services/Tasks.php | 4 ++-- .../{MaintenanceRules.php => Interval.php} | 22 +++++++++---------- .../Platform/Workers/Certificates.php | 1 + 8 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 bin/interval delete mode 100644 bin/maintenance-rules rename src/Appwrite/Platform/Tasks/{MaintenanceRules.php => Interval.php} (85%) diff --git a/.env b/.env index be0e5df718..4dd381cb82 100644 --- a/.env +++ b/.env @@ -101,8 +101,8 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL=60 -_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL=86400 +_APP_INTERVAL_DOMAIN_VERIFICATION=60 +_APP_INTERVAL_CERTIFICATE_RENEWAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index 71baa9e1c6..ecc5112cc4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -57,8 +57,8 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ + chmod +x /usr/local/bin/interval && \ chmod +x /usr/local/bin/maintenance && \ - chmod +x /usr/local/bin/maintenance-rules && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ diff --git a/bin/interval b/bin/interval new file mode 100644 index 0000000000..e4355b1dc3 --- /dev/null +++ b/bin/interval @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php interval $@ \ No newline at end of file diff --git a/bin/maintenance-rules b/bin/maintenance-rules deleted file mode 100644 index 666e517ca0..0000000000 --- a/bin/maintenance-rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php maintenance-rules $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 30b0737543..70650b0e57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -787,10 +787,10 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES - appwrite-task-maintenance-rules: - entrypoint: maintenance-rules + appwrite-task-interval: + entrypoint: interval <<: *x-logging - container_name: appwrite-task-maintenance-rules + container_name: appwrite-task-interval image: appwrite-dev networks: - appwrite @@ -822,8 +822,8 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL - - _APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL + - _APP_INTERVAL_DOMAIN_VERIFICATION + - _APP_INTERVAL_CERTIFICATE_RENEWAL appwrite-task-stats-resources: container_name: appwrite-task-stats-resources diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index f585557690..a7854e5cb6 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; -use Appwrite\Platform\Tasks\MaintenanceRules; +use Appwrite\Platform\Tasks\Interval; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\ScheduleExecutions; @@ -29,8 +29,8 @@ class Tasks extends Service $this ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) + ->addAction(Interval::getName(), new Interval()) ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(MaintenanceRules::getName(), new MaintenanceRules()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/MaintenanceRules.php b/src/Appwrite/Platform/Tasks/Interval.php similarity index 85% rename from src/Appwrite/Platform/Tasks/MaintenanceRules.php rename to src/Appwrite/Platform/Tasks/Interval.php index cbcd538d8c..74ab9db1f1 100644 --- a/src/Appwrite/Platform/Tasks/MaintenanceRules.php +++ b/src/Appwrite/Platform/Tasks/Interval.php @@ -12,17 +12,17 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\System\System; -class MaintenanceRules extends Action +class Interval extends Action { public static function getName(): string { - return 'maintenance-rules'; + return 'interval'; } public function __construct() { $this - ->desc('Schedules periodic tasks for rule verification and certificate renewal') + ->desc('Schedules tasks on regular intervals by publishing them to our queues') ->inject('dbForPlatform') ->inject('queueForCertificates') ->callback($this->action(...)); @@ -30,22 +30,22 @@ class MaintenanceRules extends Action public function action(Database $dbForPlatform, Certificate $queueForCertificates): void { - Console::title('Rule maintenance V1'); - Console::success(APP_NAME . ' rule maintenance process v1 has started'); + Console::title('Interval V1'); + Console::success(APP_NAME . ' interval process v1 has started'); - $intervalRuleDomainVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL', '60'); // 1 minute - $intervalRuleCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day + $intervalDomainVerification = (int) System::getEnv('_APP_INTERVAL_DOMAIN_VERIFICATION', '60'); // 1 minute + $intervalCertificateRenewal = (int) System::getEnv('_APP_INTERVAL_CERTIFICATE_RENEWAL', '86400'); // 1 day - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleDomainVerification) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->verifyDomain($dbForPlatform, $queueForCertificates); - }, $intervalRuleDomainVerification); + }, $intervalDomainVerification); }); - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleCertificateRenewal) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->renewCertificates($dbForPlatform, $queueForCertificates); - }, $intervalRuleCertificateRenewal); + }, $intervalCertificateRenewal); }); } diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 6371f6c313..0c4d495724 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -264,6 +264,7 @@ class Certificates extends Action // Rule not found (or) not in the expected state if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { Console::warning('Certificate generation for ' . $domain->get() . ' is skipped as the associated rule is either empty or not in the expected state.'); + return; } // Get associated certificate for the rule From 58751bbdf1990d9db8484049a30271dc84ca9383 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Tue, 23 Dec 2025 16:49:35 +0530 Subject: [PATCH 13/15] lint --- src/Appwrite/Platform/Services/Tasks.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index a7854e5cb6..941530d7ed 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -4,8 +4,8 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; -use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Interval; +use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\ScheduleExecutions; From 1b70bc812b6aa8f101e951430de91aa17cb63c7f Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Mon, 29 Dec 2025 14:51:59 +0530 Subject: [PATCH 14/15] keep certificate renewal in maintenance worker --- .env | 1 - docker-compose.yml | 1 - src/Appwrite/Platform/Tasks/Interval.php | 51 --------------------- src/Appwrite/Platform/Tasks/Maintenance.php | 45 ++++++++++++++++++ 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/.env b/.env index 19ee65350b..88dec63b1c 100644 --- a/.env +++ b/.env @@ -102,7 +102,6 @@ _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_INTERVAL_DOMAIN_VERIFICATION=60 -_APP_INTERVAL_CERTIFICATE_RENEWAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/docker-compose.yml b/docker-compose.yml index 4adbd096ab..805be67340 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -823,7 +823,6 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - _APP_INTERVAL_DOMAIN_VERIFICATION - - _APP_INTERVAL_CERTIFICATE_RENEWAL appwrite-task-stats-resources: container_name: appwrite-task-stats-resources diff --git a/src/Appwrite/Platform/Tasks/Interval.php b/src/Appwrite/Platform/Tasks/Interval.php index 74ab9db1f1..2aa15bf7d0 100644 --- a/src/Appwrite/Platform/Tasks/Interval.php +++ b/src/Appwrite/Platform/Tasks/Interval.php @@ -34,19 +34,12 @@ class Interval extends Action Console::success(APP_NAME . ' interval process v1 has started'); $intervalDomainVerification = (int) System::getEnv('_APP_INTERVAL_DOMAIN_VERIFICATION', '60'); // 1 minute - $intervalCertificateRenewal = (int) System::getEnv('_APP_INTERVAL_CERTIFICATE_RENEWAL', '86400'); // 1 day \go(function () use ($dbForPlatform, $queueForCertificates, $intervalDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->verifyDomain($dbForPlatform, $queueForCertificates); }, $intervalDomainVerification); }); - - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { - Console::loop(function () use ($dbForPlatform, $queueForCertificates) { - $this->renewCertificates($dbForPlatform, $queueForCertificates); - }, $intervalCertificateRenewal); - }); } private function verifyDomain(Database $dbForPlatform, Certificate $queueForCertificate): void @@ -79,48 +72,4 @@ class Interval extends Action ->trigger(); } } - - private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void - { - $time = DatabaseDateTime::now(); - - $certificates = $dbForPlatform->find('certificates', [ - Query::lessThan('attempts', 5), // Maximum 5 attempts - Query::isNotNull('renewDate'), - Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) - Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) - ]); - - if (\count($certificates) === 0) { - Console::info("[{$time}] No certificates for renewal."); - return; - } - - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - $appRegion = System::getEnv('_APP_REGION', 'default'); - - foreach ($certificates as $certificate) { - $domain = $certificate->getAttribute('domain'); - $rule = $isMd5 ? - $dbForPlatform->getDocument('rules', md5($domain)) : - $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - Query::limit(1) - ]); - - if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { - continue; - } - - $queueForCertificate - ->setDomain(new Document([ - 'domain' => $rule->getAttribute('domain'), - 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), - ])) - ->setAction(Certificate::ACTION_GENERATION) - ->trigger(); - } - } } diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 66d3a3d9de..c0914c6544 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -92,6 +92,7 @@ class Maintenance extends Action ->trigger(); $this->notifyDeleteConnections($queueForDeletes); + $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); $this->notifyDeleteCSVExports($queueForDeletes); @@ -113,6 +114,50 @@ class Maintenance extends Action ->trigger(); } + private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void + { + $time = DatabaseDateTime::now(); + + $certificates = $dbForPlatform->find('certificates', [ + Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), + Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) + Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) + ]); + + if (\count($certificates) === 0) { + Console::info("[{$time}] No certificates for renewal."); + return; + } + + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $appRegion = System::getEnv('_APP_REGION', 'default'); + + foreach ($certificates as $certificate) { + $domain = $certificate->getAttribute('domain'); + $rule = $isMd5 ? + $dbForPlatform->getDocument('rules', md5($domain)) : + $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain]), + Query::limit(1) + ]); + + if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { + continue; + } + + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + } + } + private function notifyDeleteCache($interval, Delete $queueForDeletes): void { $queueForDeletes From 2b96d60c1e8babc8294517693761bf9b7c69af84 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Mon, 29 Dec 2025 16:01:19 +0530 Subject: [PATCH 15/15] copilot - code quality --- .../Certificates/Exception/CertificateStatus.php | 2 +- src/Appwrite/Certificates/LetsEncrypt.php | 2 +- src/Appwrite/Platform/Tasks/Interval.php | 4 ++-- src/Appwrite/Platform/Workers/Certificates.php | 9 +++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php index 3d94109d0e..ca15a95ed8 100644 --- a/src/Appwrite/Certificates/Exception/CertificateStatus.php +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -1,6 +1,6 @@ setDomain(new Document([ 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 0c4d495724..5132687279 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -116,8 +116,6 @@ class Certificates extends Action default: throw new Exception('Invalid action: ' . $action); } - - } /** @@ -143,7 +141,7 @@ class Certificates extends Action Realtime $queueForRealtime, Certificate $queueForCertificates, Log $log, - ?string $validationDomain = null, + ?string $validationDomain = null ): void { // Get rule $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' @@ -177,7 +175,6 @@ class Certificates extends Action $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } - // Issue a TLS certificate when domain is verified if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { $queueForCertificates @@ -317,7 +314,7 @@ class Certificates extends Action $date = \date('H:i:s'); $errorMessage = "\033[90m[{$date}] \033[31mCertificate generation failed: \033[0m\n"; - $attempts = $certificate->getAttribute('attempts', 0) + 1; // // Increase attempts count + $attempts = $certificate->getAttribute('attempts', 0) + 1; // Increase attempts count // Update attributes on certificate document $certificate->setAttributes([ @@ -379,7 +376,7 @@ class Certificates extends Action /** * Update all existing domain documents so they have relation to correct certificate document. - * This solved issues: + * This solves issues: * - when adding a domain for which there is already a certificate * - when renew creates new document? It might? * - overall makes it more reliable