From 70a7deaa3807311c3925e6b7866ea9b7424b9b04 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Wed, 17 Dec 2025 20:46:54 +0530 Subject: [PATCH 01/25] Refactor certificate generation in worker --- docker-compose.yml | 1 + src/Appwrite/Certificates/Adapter.php | 2 + src/Appwrite/Certificates/LetsEncrypt.php | 5 + .../Modules/Proxy/{Http/Rules => }/Action.php | 2 +- .../Modules/Proxy/Http/Rules/API/Create.php | 2 +- .../Proxy/Http/Rules/Function/Create.php | 2 +- .../Proxy/Http/Rules/Redirect/Create.php | 2 +- .../Modules/Proxy/Http/Rules/Site/Create.php | 2 +- .../Proxy/Http/Rules/Verification/Update.php | 2 +- .../Platform/Workers/Certificates.php | 309 ++++++++---------- 10 files changed, 148 insertions(+), 181 deletions(-) rename src/Appwrite/Platform/Modules/Proxy/{Http/Rules => }/Action.php (99%) diff --git a/docker-compose.yml b/docker-compose.yml index 57007c4efc..f7e8df25e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -555,6 +555,7 @@ services: - _APP_DOMAIN_TARGET_CAA - _APP_DNS - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES - _APP_EMAIL_CERTIFICATES - _APP_REDIS_HOST - _APP_REDIS_PORT diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index ab673e9cfe..121542baa1 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -8,6 +8,8 @@ interface Adapter { public function issueCertificate(string $certName, string $domain, ?string $domainType): ?string; + public function isInstantGeneration(): bool; + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; public function deleteCertificate(string $domain): void; diff --git a/src/Appwrite/Certificates/LetsEncrypt.php b/src/Appwrite/Certificates/LetsEncrypt.php index 76638d9816..14a61203a1 100644 --- a/src/Appwrite/Certificates/LetsEncrypt.php +++ b/src/Appwrite/Certificates/LetsEncrypt.php @@ -84,6 +84,11 @@ class LetsEncrypt implements Adapter return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); } + public function isInstantGeneration(): bool + { + return true; + } + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool { $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php b/src/Appwrite/Platform/Modules/Proxy/Action.php similarity index 99% rename from src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php rename to src/Appwrite/Platform/Modules/Proxy/Action.php index 5ec5b8b8f5..c3fa535a5c 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Action.php +++ b/src/Appwrite/Platform/Modules/Proxy/Action.php @@ -1,6 +1,6 @@ desc('Certificates worker') ->inject('message') @@ -93,13 +92,12 @@ class Certificates extends Action $document = new Document($payload['domain'] ?? []); $domain = new Domain($document->getAttribute('domain', '')); + $domainType = $document->getAttribute('domainType'); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; $validationDomain = $payload['validationDomain'] ?? null; $log->addTag('domain', $domain->get()); - $domainType = $document->getAttribute('domainType'); - $this->execute($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); } @@ -163,26 +161,43 @@ class Certificates extends Action * Note: Renewals are checked and scheduled from maintenance worker */ - // Get current certificate - $certificate = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain->get()])]); + // Get rule document for domain + // TODO: (@Meldiron) Remove after 1.7.x migration + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); - // If we don't have certificate for domain yet, let's create new document. At the end we save it + // Rule not found (or) not in the expected state + if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { + Console::warning('Certificate generation for ' . $domain . ' is skipped as the associated rule is either empty or not in the expected state.'); + } + + // Get associated certificate for the rule + $certificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + // If we don't have certificate for the rule yet, let's create one. if ($certificate->isEmpty()) { $certificate = new Document(); $certificate->setAttribute('domain', $domain->get()); } - $success = false; - try { $date = \date('H:i:s'); $certificate->setAttribute('logs', "\033[90m[{$date}] \033[97mCertificate generation started. \033[0m\n"); + // Persist ASAP so that logs are reset in retry flow and user can see the latest logs on Console. + $certificate = $this->upsertCertificate($rule, $certificate, $dbForPlatform); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { $mainDomain = $validationDomain ?? $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain, $log); + $this->validateDomain($rule, $isMainDomain, $log); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -191,85 +206,148 @@ class Certificates extends Action } } - // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + // Prepare unique cert name. Using this helps prevent mismatch in configuration when renewing certificates. $certName = ID::unique(); $renewDate = $certificates->issueCertificate($certName, $domain->get(), $domainType); - // Command succeeded, store all data into document - $certificate->setAttribute('logs', 'Certificate successfully generated.'); + // If certificate is generated instantly, we can mark the rule as 'verified'. + if ($certificates->isInstantGeneration()) { + $rule->setAttribute('status', RULE_STATUS_VERIFIED); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); + } - // Update certificate info stored in database - $certificate->setAttribute('renewDate', $renewDate); - $certificate->setAttribute('attempts', 0); - $certificate->setAttribute('issueDate', DateTime::now()); - $success = true; + $certificate->setAttributes([ + 'attempts' => 0, // Reset attempts count + 'issueDate' => DateTime::now(), // Store current time as issue date + 'renewDate' => $renewDate, + ]); } catch (Throwable $e) { $logs = $e->getMessage(); $currentLogs = $certificate->getAttribute('logs', ''); $date = \date('H:i:s'); $errorMessage = "\033[90m[{$date}] \033[31mCertificate generation failed: \033[0m\n"; - $certificate->setAttribute('logs', $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000));// Limit to 500kb + $attempts = $certificate->getAttribute('attempts', 0) + 1; // // Increase attempts count - // Increase attempts count - $attempts = $certificate->getAttribute('attempts', 0) + 1; - $certificate->setAttribute('attempts', $attempts); + // Update attributes on certificate document + $certificate->setAttributes([ + 'logs' => $currentLogs . $errorMessage . \mb_strcut($logs, 0, 500000), // Limit to 500kb + 'attempts' => $attempts, + 'renewDate' => DateTime::now(), // Store current time as renew date to ensure another attempt in next maintenance cycle. + ]); - // Store current time as renew date to ensure another attempt in next maintenance cycle. - $certificate->setAttribute('renewDate', DateTime::now()); + // Mark rule as 'unverified' + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATION_FAILED); // Send email to security email $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails, $plan); throw $e; } finally { - // All actions result in new updatedAt date + // All actions result in new 'updated' date $certificate->setAttribute('updated', DateTime::now()); + // Save certificate document to database + $this->upsertCertificate($rule, $certificate, $dbForPlatform); - // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + // Ensure certificate is associated with the rule + $rule->setAttribute('certificateId', $certificate->getId()); + // Update rule and emit events + $this->updateDomainDocuments($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } /** - * Save certificate data into database. + * Save certificate data to database. * - * @param string $domain Domain name that certificate is for + * @param Document $rule Rule associated with the domain * @param Document $certificate Certificate document that we need to save - * @param bool $success * @param Database $dbForPlatform Database connection for console - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @return void + * @return Document * @throws \Utopia\Database\Exception * @throws Authorization * @throws Conflict * @throws Structure */ - private function saveCertificateDocument( - string $domain, + private function upsertCertificate( + Document $rule, Document $certificate, - bool $success, + Database $dbForPlatform, + ): Document { + // Decide whether update (or) insert is needed + $existingCertificate = $dbForPlatform->getDocument('certificates', $rule->getAttribute('certificateId') ?? ''); + + if ($existingCertificate->isEmpty()) { + $certificate->removeAttribute('$sequence'); + $certificate = $dbForPlatform->createDocument('certificates', $certificate); + } else { + $certificate = new Document(\array_merge($existingCertificate->getArrayCopy(), $certificate->getArrayCopy())); + $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); + } + + return $certificate; + } + + /** + * Update all existing domain documents so they have relation to correct certificate document. + * This solved issues: + * - when adding a domain for which there is already a certificate + * - when renew creates new document? It might? + * - overall makes it more reliable + * + * @param Document $rule Rule document that is affected by new certificate + * @param Database $dbForPlatform Database connection for console + * @param Event $queueForEvents Event publisher for events + * @param Webhook $queueForWebhooks Webhook publisher for webhooks + * @param Func $queueForFunctions Function publisher for functions + * @param Realtime $queueForRealtime Realtime publisher for realtime events + * + * @return void + */ + private function updateDomainDocuments( + Document $rule, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime ): void { - // Check if update or insert required - $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); - if (!$certificateDocument->isEmpty()) { - // Merge new data with current data - $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); - $certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate); - } else { - $certificate->removeAttribute('$sequence'); - $certificate = $dbForPlatform->createDocument('certificates', $certificate); + $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); + + $projectId = $rule->getAttribute('projectId'); + + // Skip events for console project (triggered by auto-ssl generation for 1 click setups) + if ($projectId === 'console') { + return; } - $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + $project = $dbForPlatform->getDocument('projects', $projectId); + + if ($project->isEmpty()) { + return; + } + + $ruleModel = new Rule(); + $queueForEvents + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + + /** Trigger Webhook */ + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + + /** Trigger Functions */ + $queueForFunctions + ->from($queueForEvents) + ->trigger(); + + /** Trigger Realtime Events */ + $queueForRealtime + ->setSubscribers(['console', $projectId]) + ->from($queueForEvents) + ->trigger(); } /** @@ -292,62 +370,17 @@ class Certificates extends Action * - Domain needs to be public and valid (prevents NFT domains that are not supported) * - Domain must have proper DNS record * - * @param Domain $domain Domain which we validate + * @param Document $rule Rule to validate * @param bool $isMainDomain In case of master domain, we look for different DNS configurations + * @param Log $log Logger for adding metrics * * @return void * @throws Exception */ - private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void + private function validateDomain(Document $rule, bool $isMainDomain, Log $log): void { - if (empty($domain->get())) { - throw new Exception('Missing certificate domain.'); - } - - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain.'); - } - if (!$isMainDomain) { - $validationStart = \microtime(true); - - $validators = []; - $targetCNAME = new Domain(System::getEnv('_APP_DOMAIN_TARGET_CNAME', '')); - if ($targetCNAME->isKnown() && !$targetCNAME->isTest()) { - $validators[] = new DNS($targetCNAME->get(), Record::TYPE_CNAME); - } - if ((new IP(IP::V4))->isValid(System::getEnv('_APP_DOMAIN_TARGET_A', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_A', ''), Record::TYPE_A); - } - if ((new IP(IP::V6))->isValid(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''))) { - $validators[] = new DNS(System::getEnv('_APP_DOMAIN_TARGET_AAAA', ''), Record::TYPE_AAAA); - } - - // Validate if domain target is properly configured - if (empty($validators)) { - throw new Exception('At least one of domain targets environment variable must be configured.'); - } - - // Verify domain with DNS records - $validator = new AnyOf($validators, AnyOf::TYPE_STRING); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - throw new Exception('Failed to verify domain DNS records.'); - } - - // Ensure CAA won't block certificate issuance - if (!empty(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''))) { - $validationStart = \microtime(true); - $validator = new DNS(System::getEnv('_APP_DOMAIN_TARGET_CAA', ''), Record::TYPE_CAA); - if (!$validator->isValid($domain->get())) { - $log->addExtra('dnsTimingCaa', \strval(\microtime(true) - $validationStart)); - $log->addTag('dnsDomain', $domain->get()); - $error = $validator->getDescription(); - $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error)); - throw new Exception('Failed to verify domain DNS records. CAA records do not allow Appwrite\'s certificate issuer.'); - } - } + $this->verifyRule($rule, $log); } else { // Main domain validation // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? @@ -406,78 +439,4 @@ class Certificates extends Action ->setRecipient(System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))) ->trigger(); } - - /** - * Update all existing domain documents so they have relation to correct certificate document. - * This solved issues: - * - when adding a domain for which there is already a certificate - * - when renew creates new document? It might? - * - overall makes it more reliable - * - * @param string $certificateId ID of a new or updated certificate document - * @param string $domain Domain that is affected by new certificate - * @param bool $success Was certificate generation successful? - * - * @return void - */ - private function updateDomainDocuments( - string $certificateId, - string $domain, - bool $success, - Database $dbForPlatform, - Event $queueForEvents, - Webhook $queueForWebhooks, - Func $queueForFunctions, - Realtime $queueForRealtime - ): void { - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - ]); - - if (!$rule->isEmpty()) { - $rule->setAttribute('certificateId', $certificateId); - $rule->setAttribute('status', $success ? 'verified' : 'unverified'); - $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); - - $projectId = $rule->getAttribute('projectId'); - - // Skip events for console project (triggered by auto-ssl generation for 1 click setups) - if ($projectId === 'console') { - return; - } - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - return; - } - - $ruleModel = new Rule(); - $queueForEvents - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); - - /** Trigger Webhook */ - $queueForWebhooks - ->from($queueForEvents) - ->trigger(); - - /** Trigger Functions */ - $queueForFunctions - ->from($queueForEvents) - ->trigger(); - - /** Trigger Realtime Events */ - $queueForRealtime - ->from($queueForEvents) - ->setSubscribers(['console', $projectId]) - ->trigger(); - } - } } From 0e27088e2aa3457fca73909798055df2574ee4f1 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Wed, 17 Dec 2025 20:54:18 +0530 Subject: [PATCH 02/25] change function signature --- src/Appwrite/Certificates/Adapter.php | 2 +- src/Appwrite/Certificates/LetsEncrypt.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index 121542baa1..770d2bb71d 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -8,7 +8,7 @@ interface Adapter { public function issueCertificate(string $certName, string $domain, ?string $domainType): ?string; - public function isInstantGeneration(): bool; + public function isInstantGeneration(string $domain, ?string $domainType): bool; public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; diff --git a/src/Appwrite/Certificates/LetsEncrypt.php b/src/Appwrite/Certificates/LetsEncrypt.php index 14a61203a1..7e71080a3c 100644 --- a/src/Appwrite/Certificates/LetsEncrypt.php +++ b/src/Appwrite/Certificates/LetsEncrypt.php @@ -84,7 +84,7 @@ class LetsEncrypt implements Adapter return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); } - public function isInstantGeneration(): bool + public function isInstantGeneration(string $domain, ?string $domainType): bool { return true; } From db0dbeb27b0d310d3ce1670b32b498920b2e2a88 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Wed, 17 Dec 2025 21:04:15 +0530 Subject: [PATCH 03/25] simplify --- .../Platform/Workers/Certificates.php | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 5f839f3850..68572961aa 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -195,9 +195,7 @@ class Certificates extends Action // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { - $mainDomain = $validationDomain ?? $this->getMainDomain(); - $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($rule, $isMainDomain, $log); + $this->validateDomain($rule, $domain, $validationDomain, $log); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -211,7 +209,7 @@ class Certificates extends Action $renewDate = $certificates->issueCertificate($certName, $domain->get(), $domainType); // If certificate is generated instantly, we can mark the rule as 'verified'. - if ($certificates->isInstantGeneration()) { + if ($certificates->isInstantGeneration($domain->get(), $domainType)) { $rule->setAttribute('status', RULE_STATUS_VERIFIED); $certificate->setAttribute('logs', 'Certificate successfully generated.'); } @@ -350,6 +348,31 @@ class Certificates extends Action ->trigger(); } + /** + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) + * - Domain must have proper DNS record + * + * @param Document $rule Rule to validate + * @param Domain $domain Domain to validate + * @param string|null $validationDomain Override for main domain check + * @param Log $log Logger for adding metrics + * + * @return void + * @throws Exception + */ + private function validateDomain(Document $rule, Domain $domain, ?string $validationDomain = null, Log $log): void + { + $mainDomain = $validationDomain ?? $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + if (!$isMainDomain) { + $this->verifyRule($rule, $log); + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + } + } + /** * Get main domain. Needed as we do different checks for main and non-main domains. * @@ -366,29 +389,7 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported) - * - Domain must have proper DNS record - * - * @param Document $rule Rule to validate - * @param bool $isMainDomain In case of master domain, we look for different DNS configurations - * @param Log $log Logger for adding metrics - * - * @return void - * @throws Exception - */ - private function validateDomain(Document $rule, bool $isMainDomain, Log $log): void - { - if (!$isMainDomain) { - $this->verifyRule($rule, $log); - } else { - // Main domain validation - // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? - } - } - - /** - * Method to make sure information about error is delivered to admnistrator. + * Method to make sure information about error is delivered to administrator. * * @param string $domain Domain that caused the error * @param string $errorMessage Verbose error message From 88bd35ce9881df475ceeb36cdba18af4e5132499 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 12:40:24 +0530 Subject: [PATCH 04/25] periodic task for rule verification & certificate generation --- .env | 2 + Dockerfile | 1 + bin/rules | 3 + docker-compose.yml | 37 +++++ src/Appwrite/Event/Certificate.php | 25 ++++ .../Modules/Proxy/Http/Rules/API/Create.php | 1 + .../Proxy/Http/Rules/Function/Create.php | 1 + .../Platform/Modules/Proxy/Http/Rules/Get.php | 1 + .../Proxy/Http/Rules/Redirect/Create.php | 1 + .../Modules/Proxy/Http/Rules/Site/Create.php | 1 + .../Modules/Proxy/Http/Rules/XList.php | 1 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/Maintenance.php | 42 ------ src/Appwrite/Platform/Tasks/Rules.php | 126 ++++++++++++++++++ .../Platform/Workers/Certificates.php | 97 +++++++++++++- 15 files changed, 292 insertions(+), 49 deletions(-) create mode 100644 bin/rules create mode 100644 src/Appwrite/Platform/Tasks/Rules.php diff --git a/.env b/.env index 64fc7ef10f..d211f0ae6e 100644 --- a/.env +++ b/.env @@ -101,6 +101,8 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 +_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL=60 +_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index e146008222..b65b73469c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -58,6 +58,7 @@ RUN mkdir -p /storage/uploads && \ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/rules && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ diff --git a/bin/rules b/bin/rules new file mode 100644 index 0000000000..77e195eb61 --- /dev/null +++ b/bin/rules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php rules $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f7e8df25e6..99b42561b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -786,6 +786,43 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES + appwrite-task-rules: + entrypoint: rules + <<: *x-logging + container_name: appwrite-task-rules + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_DOMAIN + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A + - _APP_DOMAIN_TARGET_CAA + - _APP_DNS + - _APP_DOMAIN_FUNCTIONS + - _APP_DOMAIN_SITES + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_DATABASE_SHARED_TABLES + - _APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL + - _APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL + appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index 00875c7a4a..7e60d15180 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -7,7 +7,10 @@ use Utopia\Queue\Publisher; class Certificate extends Event { + public const string ACTION_DOMAIN_VERIFICATION = 'verification'; + public const string ACTION_GENERATION = 'generation'; protected bool $skipRenewCheck = false; + protected string $action = self::ACTION_GENERATION; protected ?Document $domain = null; protected ?string $validationDomain = null; @@ -90,6 +93,28 @@ class Certificate extends Event return $this->skipRenewCheck; } + /** + * Set action for this certificate event. + * + * @param string $action + * @return self + */ + public function setAction(string $action): self + { + $this->action = $action; + return $this; + } + + /** + * Get action for this certificate event. + * + * @return string + */ + public function getAction(): string + { + return $this->action; + } + /** * Prepare the payload for the event diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php index ea2f34f8fd..95ea8dd8cf 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php @@ -125,6 +125,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php index 0009a2eb57..ea0fb69050 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php index 4581cb3d08..4c17fdc460 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php index 493c827c46..f21374b49a 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php @@ -147,6 +147,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php index 9ec79139af..26bd453eb3 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php @@ -143,6 +143,7 @@ class Create extends Action 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), ])) + ->setAction(Certificate::ACTION_GENERATION) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php index 198bf55a6f..e160b71060 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Proxy\Http\Rules; use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 3ada193cf7..3b0ba7d5ea 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -7,6 +7,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; +use Appwrite\Platform\Tasks\Rules; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; @@ -29,6 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) + ->addAction(Rules::getName(), new Rules()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 9c88bc4d4e..66d3a3d9de 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -92,7 +92,6 @@ class Maintenance extends Action ->trigger(); $this->notifyDeleteConnections($queueForDeletes); - $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); $this->notifyDeleteCSVExports($queueForDeletes); @@ -114,47 +113,6 @@ class Maintenance extends Action ->trigger(); } - private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void - { - $time = DatabaseDateTime::now(); - - $certificates = $dbForPlatform->find('certificates', [ - Query::lessThan('attempts', 5), // Maximum 5 attempts - Query::isNotNull('renewDate'), - Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) - Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) - ]); - - - if (\count($certificates) > 0) { - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - - foreach ($certificates as $certificate) { - $domain = $certificate->getAttribute('domain'); - $rule = $isMd5 - ? $dbForPlatform->getDocument('rules', md5($domain)) - : $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - ]); - - if ($rule->isEmpty() || $rule->getAttribute('region') !== System::getEnv('_APP_REGION', 'default')) { - continue; - } - - $queueForCertificate - ->setDomain(new Document([ - 'domain' => $certificate->getAttribute('domain') - ])) - ->trigger(); - } - } else { - Console::info("[{$time}] No certificates for renewal."); - } - } - private function notifyDeleteCache($interval, Delete $queueForDeletes): void { $queueForDeletes diff --git a/src/Appwrite/Platform/Tasks/Rules.php b/src/Appwrite/Platform/Tasks/Rules.php new file mode 100644 index 0000000000..1398459a88 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Rules.php @@ -0,0 +1,126 @@ +desc('Schedules periodic tasks for rule verification and certificate renewal') + ->inject('dbForPlatform') + ->inject('queueForCertificates') + ->callback($this->action(...)); + } + + public function action(Database $dbForPlatform, Certificate $queueForCertificates): void + { + Console::title('Interval V1'); + Console::success(APP_NAME . ' interval process v1 has started'); + + $intervalRuleVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL', '60'); // 1 minute + $intervalCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day + + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleVerification) { + Console::loop(function () use ($dbForPlatform, $queueForCertificates) { + $this->checkRuleVerification($dbForPlatform, $queueForCertificates); + }, $intervalRuleVerification); + }); + + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { + Console::loop(function () use ($dbForPlatform, $queueForCertificates) { + $this->renewCertificates($dbForPlatform, $queueForCertificates); + }, $intervalCertificateRenewal); + }); + } + + private function checkRuleVerification(Database $dbForPlatform, Certificate $queueForCertificate): void + { + $time = DatabaseDateTime::now(); + $fromTime = new DateTime('-3 days'); // Max 3 days old + + $rules = $dbForPlatform->find('rules', [ + Query::createdAfter(DatabaseDateTime::format($fromTime)), + Query::equal('status', [RULE_STATUS_CREATED]), // Created but not verified yet + Query::orderAsc('$updatedAt'), // Pick the ones waiting for another attempt for longest + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), // Only current region + Query::limit(30), // Reasonable pagination limit, processable within a minute + ]); + + if (\count($rules) === 0) { + Console::info("[{$time}] No rules for verification."); + return; // No rules to verify + } + + Console::info("[{$time}] Found " . \count($rules) . " rules for verification, scheduling jobs."); + + foreach ($rules as $rule) { + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_DOMAIN_VERIFICATION) + ->trigger(); + } + } + + private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void + { + $time = DatabaseDateTime::now(); + + $certificates = $dbForPlatform->find('certificates', [ + Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), + Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) + Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) + ]); + + if (\count($certificates) === 0) { + Console::info("[{$time}] No certificates for renewal."); + return; + } + + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $appRegion = System::getEnv('_APP_REGION', 'default'); + + foreach ($certificates as $certificate) { + $domain = $certificate->getAttribute('domain'); + $rule = $isMd5 ? + $dbForPlatform->getDocument('rules', md5($domain)) : + $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain]), + Query::limit(1) + ]); + + if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { + continue; + } + + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + } + } +} diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 68572961aa..a78918471a 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -3,11 +3,13 @@ namespace Appwrite\Platform\Workers; use Appwrite\Certificates\Adapter as CertificatesAdapter; +use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; +use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Platform\Modules\Proxy\Action; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; @@ -20,9 +22,9 @@ use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Structure; -use Utopia\Database\Validator\Authorization as ValidatorAuthorization; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization as ValidatorAuthorization; use Utopia\Domains\Domain; use Utopia\Locale\Locale; use Utopia\Logger\Log; @@ -52,6 +54,7 @@ class Certificates extends Action ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') + ->inject('queueForCertificates') ->inject('log') ->inject('certificates') ->inject('plan') @@ -66,6 +69,7 @@ class Certificates extends Action * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime + * @param Certificate $queueForCertificates * @param Log $log * @param CertificatesAdapter $certificates * @return void @@ -80,6 +84,7 @@ class Certificates extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, + Certificate $queueForCertificates, Log $log, CertificatesAdapter $certificates, array $plan @@ -95,10 +100,88 @@ class Certificates extends Action $domainType = $document->getAttribute('domainType'); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; $validationDomain = $payload['validationDomain'] ?? null; + $action = $payload['action'] ?? Certificate::ACTION_GENERATION; $log->addTag('domain', $domain->get()); - $this->execute($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + switch ($action) { + case Certificate::ACTION_DOMAIN_VERIFICATION: + $this->handleDomainVerificationAction($domain, $dbForPlatform, $log, $queueForCertificates, $validationDomain); + break; + + case Certificate::ACTION_GENERATION: + $this->handleCertificateGenerationAction($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain); + break; + + default: + throw new Exception('Invalid action: ' . $action); + } + + + } + + /** + * @param Domain $domain + * @param Database $dbForPlatform + * @param Log $log + * @param Certificate $queueForCertificates + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + private function handleDomainVerificationAction( + Domain $domain, + Database $dbForPlatform, + Log $log, + Certificate $queueForCertificates, + ?string $validationDomain = null, + ): void { + // Get rule + $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' + ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get()))) + : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain->get()]), + Query::limit(1), + ])); + + // Skip if rule is not desired state (created but not verified yet). + if ($rule->getAttribute('status', '') !== RULE_STATUS_CREATED) { + Console::warning('Domain verification for ' . $rule->getAttribute('domain', '') . ' is not needed.'); + return; + } + + Console::info('Domain verification for ' . $rule->getAttribute('domain', '') . ' started.'); + + $updates = new Document(); + try { + // Verify DNS records + $this->validateDomain($rule, $domain, $log, $validationDomain); + // Reset logs and status for the rule + $updates + ->setAttribute('logs', '') + ->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); + + Console::success('Domain verification succeeded.'); + } catch (AppwriteException $err) { + Console::warning('Domain verification failed: ' . $err->getMessage()); + $updates->setAttribute('logs', $err->getMessage()); + } + + echo "updating rule with updates: " . \var_dump($updates); + $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $updates); + + // Issue a TLS certificate when domain is verified + if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { + $queueForCertificates + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + + Console::success('Certificate generation triggered successfully.'); + } } /** @@ -117,7 +200,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute( + private function handleCertificateGenerationAction( Domain $domain, ?string $domainType, Database $dbForPlatform, @@ -172,7 +255,7 @@ class Certificates extends Action // Rule not found (or) not in the expected state if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { - Console::warning('Certificate generation for ' . $domain . ' is skipped as the associated rule is either empty or not in the expected state.'); + Console::warning('Certificate generation for ' . $domain->get() . ' is skipped as the associated rule is either empty or not in the expected state.'); } // Get associated certificate for the rule @@ -195,7 +278,7 @@ class Certificates extends Action // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { - $this->validateDomain($rule, $domain, $validationDomain, $log); + $this->validateDomain($rule, $domain, $log, $validationDomain); // If certificate exists already, double-check expiry date. Skip if job is forced if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) { @@ -355,13 +438,13 @@ class Certificates extends Action * * @param Document $rule Rule to validate * @param Domain $domain Domain to validate - * @param string|null $validationDomain Override for main domain check * @param Log $log Logger for adding metrics + * @param string|null $validationDomain Override for main domain check * * @return void * @throws Exception */ - private function validateDomain(Document $rule, Domain $domain, ?string $validationDomain = null, Log $log): void + private function validateDomain(Document $rule, Domain $domain, Log $log, ?string $validationDomain = null): void { $mainDomain = $validationDomain ?? $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; From 6f1d2d094a14e4972430e727cb68dd45b8c54ad5 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 13:00:49 +0530 Subject: [PATCH 05/25] rename task --- .env | 4 ++-- Dockerfile | 2 +- bin/maintenance-rules | 3 +++ bin/rules | 3 --- docker-compose.yml | 10 +++++----- src/Appwrite/Platform/Services/Tasks.php | 4 ++-- .../Tasks/{Rules.php => MaintenanceRules.php} | 16 ++++++++-------- 7 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 bin/maintenance-rules delete mode 100644 bin/rules rename src/Appwrite/Platform/Tasks/{Rules.php => MaintenanceRules.php} (89%) diff --git a/.env b/.env index d211f0ae6e..be0e5df718 100644 --- a/.env +++ b/.env @@ -101,8 +101,8 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL=60 -_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL=86400 +_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL=60 +_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index b65b73469c..71baa9e1c6 100755 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ RUN mkdir -p /storage/uploads && \ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/maintenance && \ - chmod +x /usr/local/bin/rules && \ + chmod +x /usr/local/bin/maintenance-rules && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ diff --git a/bin/maintenance-rules b/bin/maintenance-rules new file mode 100644 index 0000000000..666e517ca0 --- /dev/null +++ b/bin/maintenance-rules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php maintenance-rules $@ \ No newline at end of file diff --git a/bin/rules b/bin/rules deleted file mode 100644 index 77e195eb61..0000000000 --- a/bin/rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php rules $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 99b42561b3..5490d19b68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -786,10 +786,10 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES - appwrite-task-rules: - entrypoint: rules + appwrite-task-maintenance-rules: + entrypoint: maintenance-rules <<: *x-logging - container_name: appwrite-task-rules + container_name: appwrite-task-maintenance-rules image: appwrite-dev networks: - appwrite @@ -820,8 +820,8 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL - - _APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL + - _APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL + - _APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL appwrite-task-stats-resources: container_name: appwrite-task-stats-resources diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 3b0ba7d5ea..f68e3922ac 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -7,7 +7,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; -use Appwrite\Platform\Tasks\Rules; +use Appwrite\Platform\Tasks\MaintenanceRules; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; @@ -30,7 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(Rules::getName(), new Rules()) + ->addAction(MaintenanceRules::getName(), new MaintenanceRules()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/Rules.php b/src/Appwrite/Platform/Tasks/MaintenanceRules.php similarity index 89% rename from src/Appwrite/Platform/Tasks/Rules.php rename to src/Appwrite/Platform/Tasks/MaintenanceRules.php index 1398459a88..03e08bda0a 100644 --- a/src/Appwrite/Platform/Tasks/Rules.php +++ b/src/Appwrite/Platform/Tasks/MaintenanceRules.php @@ -12,11 +12,11 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\System\System; -class Rules extends Action +class MaintenanceRules extends Action { public static function getName(): string { - return 'rules'; + return 'maintenance-rules'; } public function __construct() @@ -33,19 +33,19 @@ class Rules extends Action Console::title('Interval V1'); Console::success(APP_NAME . ' interval process v1 has started'); - $intervalRuleVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL', '60'); // 1 minute - $intervalCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day + $intervalRuleDomainVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL', '60'); // 1 minute + $intervalRuleCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleVerification) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->checkRuleVerification($dbForPlatform, $queueForCertificates); - }, $intervalRuleVerification); + }, $intervalRuleDomainVerification); }); - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleCertificateRenewal) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->renewCertificates($dbForPlatform, $queueForCertificates); - }, $intervalCertificateRenewal); + }, $intervalRuleCertificateRenewal); }); } From 62b391ef680c6a183e1484d6425f103b46470f1e Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 13:46:56 +0530 Subject: [PATCH 06/25] certificate status exception --- src/Appwrite/Certificates/Adapter.php | 2 ++ .../Certificates/Exception/CertificateStatus.php | 10 ++++++++++ src/Appwrite/Certificates/LetsEncrypt.php | 6 ++++++ src/Appwrite/Platform/Tasks/MaintenanceRules.php | 4 ++-- 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 src/Appwrite/Certificates/Exception/CertificateStatus.php diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php index 770d2bb71d..47d865ad08 100644 --- a/src/Appwrite/Certificates/Adapter.php +++ b/src/Appwrite/Certificates/Adapter.php @@ -10,6 +10,8 @@ interface Adapter public function isInstantGeneration(string $domain, ?string $domainType): bool; + public function getCertificateStatus(string $domain, ?string $domainType): string; + public function isRenewRequired(string $domain, ?string $domainType, Log $log): bool; public function deleteCertificate(string $domain): void; diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php new file mode 100644 index 0000000000..652c1c4253 --- /dev/null +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -0,0 +1,10 @@ + Date: Thu, 18 Dec 2025 15:14:30 +0530 Subject: [PATCH 07/25] add action to payload --- docker-compose.yml | 2 ++ src/Appwrite/Event/Certificate.php | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5490d19b68..30b0737543 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -754,6 +754,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: + - mariadb - redis environment: - _APP_ENV @@ -797,6 +798,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: + - mariadb - redis environment: - _APP_ENV diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index 7e60d15180..4ad12094c2 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -127,7 +127,8 @@ class Certificate extends Event 'project' => $this->project, 'domain' => $this->domain, 'skipRenewCheck' => $this->skipRenewCheck, - 'validationDomain' => $this->validationDomain + 'validationDomain' => $this->validationDomain, + 'action' => $this->action ]; } } From aa4ecdf13898eafa3111e8301f0b88b874570caf Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 16:07:15 +0530 Subject: [PATCH 08/25] emit events --- .../Platform/Workers/Certificates.php | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index a78918471a..5c3fe9aede 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -106,7 +106,7 @@ class Certificates extends Action switch ($action) { case Certificate::ACTION_DOMAIN_VERIFICATION: - $this->handleDomainVerificationAction($domain, $dbForPlatform, $log, $queueForCertificates, $validationDomain); + $this->handleDomainVerificationAction($domain, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForCertificates, $log, $validationDomain); break; case Certificate::ACTION_GENERATION: @@ -123,8 +123,13 @@ class Certificates extends Action /** * @param Domain $domain * @param Database $dbForPlatform - * @param Log $log + * @param Event $queueForEvents + * @param Webhook $queueForWebhooks + * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param Certificate $queueForCertificates + * @param Log $log + * @param string|null $validationDomain * @return void * @throws Throwable * @throws \Utopia\Database\Exception @@ -132,8 +137,12 @@ class Certificates extends Action private function handleDomainVerificationAction( Domain $domain, Database $dbForPlatform, - Log $log, + Event $queueForEvents, + Webhook $queueForWebhooks, + Func $queueForFunctions, + Realtime $queueForRealtime, Certificate $queueForCertificates, + Log $log, ?string $validationDomain = null, ): void { // Get rule @@ -152,23 +161,22 @@ class Certificates extends Action Console::info('Domain verification for ' . $rule->getAttribute('domain', '') . ' started.'); - $updates = new Document(); try { // Verify DNS records $this->validateDomain($rule, $domain, $log, $validationDomain); // Reset logs and status for the rule - $updates - ->setAttribute('logs', '') - ->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); + $rule->setAttribute('logs', ''); + $rule->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING); Console::success('Domain verification succeeded.'); } catch (AppwriteException $err) { Console::warning('Domain verification failed: ' . $err->getMessage()); - $updates->setAttribute('logs', $err->getMessage()); + $rule->setAttribute('logs', $err->getMessage()); + } finally { + // Update rule and emit events + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } - echo "updating rule with updates: " . \var_dump($updates); - $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $updates); // Issue a TLS certificate when domain is verified if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { @@ -333,7 +341,7 @@ class Certificates extends Action // Ensure certificate is associated with the rule $rule->setAttribute('certificateId', $certificate->getId()); // Update rule and emit events - $this->updateDomainDocuments($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); + $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } @@ -384,7 +392,7 @@ class Certificates extends Action * * @return void */ - private function updateDomainDocuments( + protected function updateRuleAndSendEvents( Document $rule, Database $dbForPlatform, Event $queueForEvents, From e78887001afb86af53311f31834bf227f2de702a Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 16:18:23 +0530 Subject: [PATCH 09/25] spacing --- src/Appwrite/Platform/Workers/Certificates.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 5c3fe9aede..6371f6c313 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -401,7 +401,6 @@ class Certificates extends Action Realtime $queueForRealtime ): void { $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $rule); - $projectId = $rule->getAttribute('projectId'); // Skip events for console project (triggered by auto-ssl generation for 1 click setups) @@ -410,7 +409,6 @@ class Certificates extends Action } $project = $dbForPlatform->getDocument('projects', $projectId); - if ($project->isEmpty()) { return; } From 16fb25ce5a9e74961e7b3a0ef40ebb47d7b34f3b Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 17:10:02 +0530 Subject: [PATCH 10/25] tiny --- src/Appwrite/Platform/Tasks/MaintenanceRules.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/MaintenanceRules.php b/src/Appwrite/Platform/Tasks/MaintenanceRules.php index c83207063a..cbcd538d8c 100644 --- a/src/Appwrite/Platform/Tasks/MaintenanceRules.php +++ b/src/Appwrite/Platform/Tasks/MaintenanceRules.php @@ -38,7 +38,7 @@ class MaintenanceRules extends Action \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { - $this->checkRuleVerification($dbForPlatform, $queueForCertificates); + $this->verifyDomain($dbForPlatform, $queueForCertificates); }, $intervalRuleDomainVerification); }); @@ -49,7 +49,7 @@ class MaintenanceRules extends Action }); } - private function checkRuleVerification(Database $dbForPlatform, Certificate $queueForCertificate): void + private function verifyDomain(Database $dbForPlatform, Certificate $queueForCertificate): void { $time = DatabaseDateTime::now(); $fromTime = new DateTime('-3 days'); // Max 3 days old @@ -63,11 +63,11 @@ class MaintenanceRules extends Action ]); if (\count($rules) === 0) { - Console::info("[{$time}] No rules for verification."); + Console::info("[{$time}] No rules for domain verification."); return; // No rules to verify } - Console::info("[{$time}] Found " . \count($rules) . " rules for verification, scheduling jobs."); + Console::info("[{$time}] Found " . \count($rules) . " rules for domain verification, scheduling jobs."); foreach ($rules as $rule) { $queueForCertificate From cf90653deb7d44ae7f42eff80725e083e3f07ff9 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Thu, 18 Dec 2025 17:30:08 +0530 Subject: [PATCH 11/25] lint --- src/Appwrite/Certificates/Exception/CertificateStatus.php | 2 +- src/Appwrite/Platform/Services/Tasks.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php index 652c1c4253..3d94109d0e 100644 --- a/src/Appwrite/Certificates/Exception/CertificateStatus.php +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -7,4 +7,4 @@ use Exception; // Exception thrown during certificate status retrieval class CertificateStatus extends Exception { -} \ No newline at end of file +} diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index f68e3922ac..f585557690 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -5,9 +5,9 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; +use Appwrite\Platform\Tasks\MaintenanceRules; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; -use Appwrite\Platform\Tasks\MaintenanceRules; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; From 47ea2893c3bb469edf3048e95004f7eda3ff149e Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Tue, 23 Dec 2025 16:32:43 +0530 Subject: [PATCH 12/25] rename task to Interval --- .env | 4 ++-- Dockerfile | 2 +- bin/interval | 3 +++ bin/maintenance-rules | 3 --- docker-compose.yml | 10 ++++----- src/Appwrite/Platform/Services/Tasks.php | 4 ++-- .../{MaintenanceRules.php => Interval.php} | 22 +++++++++---------- .../Platform/Workers/Certificates.php | 1 + 8 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 bin/interval delete mode 100644 bin/maintenance-rules rename src/Appwrite/Platform/Tasks/{MaintenanceRules.php => Interval.php} (85%) diff --git a/.env b/.env index be0e5df718..4dd381cb82 100644 --- a/.env +++ b/.env @@ -101,8 +101,8 @@ _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL=60 -_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL=86400 +_APP_INTERVAL_DOMAIN_VERIFICATION=60 +_APP_INTERVAL_CERTIFICATE_RENEWAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/Dockerfile b/Dockerfile index 71baa9e1c6..ecc5112cc4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -57,8 +57,8 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ + chmod +x /usr/local/bin/interval && \ chmod +x /usr/local/bin/maintenance && \ - chmod +x /usr/local/bin/maintenance-rules && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ diff --git a/bin/interval b/bin/interval new file mode 100644 index 0000000000..e4355b1dc3 --- /dev/null +++ b/bin/interval @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php interval $@ \ No newline at end of file diff --git a/bin/maintenance-rules b/bin/maintenance-rules deleted file mode 100644 index 666e517ca0..0000000000 --- a/bin/maintenance-rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php maintenance-rules $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 30b0737543..70650b0e57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -787,10 +787,10 @@ services: - _APP_MAINTENANCE_START_TIME - _APP_DATABASE_SHARED_TABLES - appwrite-task-maintenance-rules: - entrypoint: maintenance-rules + appwrite-task-interval: + entrypoint: interval <<: *x-logging - container_name: appwrite-task-maintenance-rules + container_name: appwrite-task-interval image: appwrite-dev networks: - appwrite @@ -822,8 +822,8 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL - - _APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL + - _APP_INTERVAL_DOMAIN_VERIFICATION + - _APP_INTERVAL_CERTIFICATE_RENEWAL appwrite-task-stats-resources: container_name: appwrite-task-stats-resources diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index f585557690..a7854e5cb6 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; -use Appwrite\Platform\Tasks\MaintenanceRules; +use Appwrite\Platform\Tasks\Interval; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\ScheduleExecutions; @@ -29,8 +29,8 @@ class Tasks extends Service $this ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) + ->addAction(Interval::getName(), new Interval()) ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(MaintenanceRules::getName(), new MaintenanceRules()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/MaintenanceRules.php b/src/Appwrite/Platform/Tasks/Interval.php similarity index 85% rename from src/Appwrite/Platform/Tasks/MaintenanceRules.php rename to src/Appwrite/Platform/Tasks/Interval.php index cbcd538d8c..74ab9db1f1 100644 --- a/src/Appwrite/Platform/Tasks/MaintenanceRules.php +++ b/src/Appwrite/Platform/Tasks/Interval.php @@ -12,17 +12,17 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\System\System; -class MaintenanceRules extends Action +class Interval extends Action { public static function getName(): string { - return 'maintenance-rules'; + return 'interval'; } public function __construct() { $this - ->desc('Schedules periodic tasks for rule verification and certificate renewal') + ->desc('Schedules tasks on regular intervals by publishing them to our queues') ->inject('dbForPlatform') ->inject('queueForCertificates') ->callback($this->action(...)); @@ -30,22 +30,22 @@ class MaintenanceRules extends Action public function action(Database $dbForPlatform, Certificate $queueForCertificates): void { - Console::title('Rule maintenance V1'); - Console::success(APP_NAME . ' rule maintenance process v1 has started'); + Console::title('Interval V1'); + Console::success(APP_NAME . ' interval process v1 has started'); - $intervalRuleDomainVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_DOMAIN_VERIFICATION_INTERVAL', '60'); // 1 minute - $intervalRuleCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_RULE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day + $intervalDomainVerification = (int) System::getEnv('_APP_INTERVAL_DOMAIN_VERIFICATION', '60'); // 1 minute + $intervalCertificateRenewal = (int) System::getEnv('_APP_INTERVAL_CERTIFICATE_RENEWAL', '86400'); // 1 day - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleDomainVerification) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->verifyDomain($dbForPlatform, $queueForCertificates); - }, $intervalRuleDomainVerification); + }, $intervalDomainVerification); }); - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleCertificateRenewal) { + \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->renewCertificates($dbForPlatform, $queueForCertificates); - }, $intervalRuleCertificateRenewal); + }, $intervalCertificateRenewal); }); } diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 6371f6c313..0c4d495724 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -264,6 +264,7 @@ class Certificates extends Action // Rule not found (or) not in the expected state if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) { Console::warning('Certificate generation for ' . $domain->get() . ' is skipped as the associated rule is either empty or not in the expected state.'); + return; } // Get associated certificate for the rule From 58751bbdf1990d9db8484049a30271dc84ca9383 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Tue, 23 Dec 2025 16:49:35 +0530 Subject: [PATCH 13/25] lint --- src/Appwrite/Platform/Services/Tasks.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index a7854e5cb6..941530d7ed 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -4,8 +4,8 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; -use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Interval; +use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\ScheduleExecutions; From da7738edaa82002f6ae6bee3191568309309b653 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 29 Dec 2025 01:50:39 +0000 Subject: [PATCH 14/25] Feat: Storage module --- .../Modules/Storage/Http/Buckets/Create.php | 167 +++++++ .../Modules/Storage/Http/Buckets/Delete.php | 89 ++++ .../Storage/Http/Buckets/Files/Create.php | 452 ++++++++++++++++++ .../Storage/Http/Buckets/Files/Delete.php | 17 + .../Http/Buckets/Files/Download/Get.php | 17 + .../Storage/Http/Buckets/Files/Get.php | 91 ++++ .../Http/Buckets/Files/Preview/Get.php | 17 + .../Storage/Http/Buckets/Files/Push/Get.php | 17 + .../Storage/Http/Buckets/Files/Update.php | 17 + .../Storage/Http/Buckets/Files/View/Get.php | 17 + .../Storage/Http/Buckets/Files/XList.php | 151 ++++++ .../Modules/Storage/Http/Buckets/Get.php | 65 +++ .../Modules/Storage/Http/Buckets/Update.php | 131 +++++ .../Modules/Storage/Http/Buckets/XList.php | 117 +++++ .../Modules/Storage/Http/Usage/Get.php | 17 + .../Modules/Storage/Http/Usage/XList.php | 17 + .../Platform/Modules/Storage/Module.php | 14 + .../Modules/Storage/Services/Http.php | 51 ++ 18 files changed, 1464 insertions(+) create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Module.php create mode 100644 src/Appwrite/Platform/Modules/Storage/Services/Http.php diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php new file mode 100644 index 0000000000..a4d28ab487 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php @@ -0,0 +1,167 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/storage/buckets') + ->desc('Create bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].create') + ->label('audits.event', 'bucket.create') + ->label('audits.resource', 'bucket/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'createBucket', + description: '/docs/references/storage/create-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('name', '', new Text(128), 'Bucket name') + ->param('permissions', null, new Nullable(new \Utopia\Database\Validator\Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) + ->param('maximumFileSize', fn(array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn(array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) + ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) + ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $name, + ?array $permissions, + bool $fileSecurity, + bool $enabled, + int $maximumFileSize, + array $allowedFileExtensions, + ?string $compression, + ?bool $encryption, + bool $antivirus, + bool $transformations, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId; + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions) ?? []; + $compression ??= Compression::NONE; + $encryption ??= true; + try { + $files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? []; + if (empty($files)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Files collection is not configured.'); + } + + $attributes = []; + $indexes = []; + + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + 'default' => $attribute['default'] ?? null, + 'format' => $attribute['format'] ?? '' + ]); + } + + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $dbForProject->createDocument('buckets', new Document([ + '$id' => $bucketId, + '$collection' => 'buckets', + '$permissions' => $permissions, + 'name' => $name, + 'maximumFileSize' => $maximumFileSize, + 'allowedFileExtensions' => $allowedFileExtensions, + 'fileSecurity' => $fileSecurity, + 'enabled' => $enabled, + 'compression' => $compression, + 'encryption' => $encryption, + 'antivirus' => $antivirus, + 'transformations' => $transformations, + 'search' => implode(' ', [$bucketId, $name]), + ])); + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + $dbForProject->createCollection('bucket_' . $bucket->getSequence(), $attributes, $indexes, permissions: $permissions, documentSecurity: $fileSecurity); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_BUCKET_ALREADY_EXISTS); + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php new file mode 100644 index 0000000000..9523f55e12 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Delete.php @@ -0,0 +1,89 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Delete bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('audits.event', 'bucket.delete') + ->label('event', 'buckets.[bucketId].delete') + ->label('audits.resource', 'bucket/{request.bucketId}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'deleteBucket', + description: '/docs/references/storage/delete-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_NOCONTENT, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + Response $response, + Database $dbForProject, + DeleteEvent $queueForDeletes, + Event $queueForEvents + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!$dbForProject->deleteDocument('buckets', $bucketId)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove bucket from DB'); + } + + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($bucket); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setPayload($response->output($bucket, Response::MODEL_BUCKET)) + ; + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php new file mode 100644 index 0000000000..8640dedb8b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -0,0 +1,452 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/storage/buckets/:bucketId/files') + ->desc('Create file') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('audits.event', 'file.create') + ->label('event', 'buckets.[bucketId].files.[fileId].create') + ->label('audits.resource', 'file/{response.$id}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'createFile', + description: '/docs/references/storage/create-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_FILE, + ) + ], + type: MethodType::UPLOAD, + requestType: ContentType::MULTIPART + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new CustomId(), 'File ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('file', [], new File(), 'Binary file. Appwrite SDKs provide helpers to handle file input. [Learn about file input](https://appwrite.io/docs/products/storage/upload-download#input-file).', skipValidation: true) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE])), 'An array of permission strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('user') + ->inject('queueForEvents') + ->inject('mode') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + mixed $file, + ?array $permissions, + Request $request, + Response $response, + Database $dbForProject, + Document $user, + Event $queueForEvents, + string $mode, + Device $deviceForFiles, + Device $deviceForLocal + ) { + $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $allowedPermissions = [ + \Utopia\Database\Database::PERMISSION_READ, + \Utopia\Database\Database::PERMISSION_UPDATE, + \Utopia\Database\Database::PERMISSION_DELETE, + ]; + + // Map aggregate permissions to into the set of individual permissions they represent. + $permissions = Permission::aggregate($permissions, $allowedPermissions); + + // Add permissions for current the user if none were provided. + if (\is_null($permissions)) { + $permissions = []; + if (!empty($user->getId()) && !$isPrivilegedUser) { + foreach ($allowedPermissions as $permission) { + $permissions[] = (new Permission($permission, 'user', $user->getId()))->toString(); + } + } + } + + // Users can only manage their own roles, API keys and Admin users can manage any + $roles = Authorization::getRoles(); + if (!$isAPIKey && !$isPrivilegedUser) { + foreach (\Utopia\Database\Database::PERMISSIONS as $type) { + foreach ($permissions as $permission) { + $permission = Permission::parse($permission); + if ($permission->getPermission() != $type) { + continue; + } + $role = (new Role( + $permission->getRole(), + $permission->getIdentifier(), + $permission->getDimension() + ))->toString(); + if (!Authorization::isRole($role)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); + } + } + } + } + + $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); + if ($maximumFileSize > (int) System::getEnv('_APP_STORAGE_LIMIT', 0)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Maximum bucket file size is larger than _APP_STORAGE_LIMIT'); + } + + $file = $request->getFiles('file'); + + // GraphQL multipart spec adds files with index keys + if (empty($file)) { + $file = $request->getFiles(0); + } + + if (empty($file)) { + throw new Exception(Exception::STORAGE_FILE_EMPTY); + } + + // Make sure we handle a single file and multiple files the same way + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + + $contentRange = $request->getHeader('content-range'); + $fileId = $fileId === 'unique()' ? ID::unique() : $fileId; + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $fileSize = $request->getContentRangeSize(); + $fileId = $request->getHeader('x-appwrite-id', $fileId); + // TODO make `end >= $fileSize` in next breaking version + if (is_null($start) || is_null($end) || is_null($fileSize) || $end > $fileSize) { + throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE); + } + + $idValidator = new UID(); + if (!$idValidator->isValid($fileId)) { + throw new Exception(Exception::STORAGE_INVALID_APPWRITE_ID); + } + + // TODO remove the condition that checks `$end === $fileSize` in next breaking version + if ($end === $fileSize - 1 || $end === $fileSize) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($fileSize / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)) + 1; + } + } + + /** + * Validators + */ + // Check if file type is allowed + $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); + $fileExt = new FileExt($allowedFileExtensions); + if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, 'File extension not allowed'); + } + + // Check if file size is exceeding allowed limit + $fileSizeValidator = new FileSize($maximumFileSize); + if (!$fileSizeValidator->isValid($fileSize)) { + throw new Exception(Exception::STORAGE_INVALID_FILE_SIZE, 'File size not allowed'); + } + + $upload = new Upload(); + if (!$upload->isValid($fileTmpName)) { + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + + // Save to storage + $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); + $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + + $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + $uploaded = $file->getAttribute('chunksUploaded', 0); + $metadata = $file->getAttribute('metadata', []); + + if ($chunk === -1) { + $chunk = $chunks; + } + + if ($uploaded === $chunks) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } + } + + $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + + if (empty($chunksUploaded)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); + } + + if ($chunksUploaded === $chunks) { + if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { + $antivirus = new Network( + System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) + ); + + if (!$antivirus->fileScan($path)) { + $deviceForFiles->delete($path); + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + } + + $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption + $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption + $data = ''; + // Compression + $algorithm = $bucket->getAttribute('compression', Compression::NONE); + if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { + $data = $deviceForFiles->read($path); + switch ($algorithm) { + case Compression::ZSTD: + $compressor = new Zstd(); + break; + case Compression::GZIP: + default: + $compressor = new GZIP(); + break; + } + $data = $compressor->compress($data); + } else { + // reset the algorithm to none as we do not compress the file + // if file size exceedes the APP_STORAGE_READ_BUFFER + // regardless the bucket compression algoorithm + $algorithm = Compression::NONE; + } + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + if (empty($data)) { + $data = $deviceForFiles->read($path); + } + $key = System::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if (!empty($data)) { + if (!$deviceForFiles->write($path, $data, $mimeType)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); + } + } + + $sizeActual = $deviceForFiles->getFileSize($path); + + $openSSLVersion = null; + $openSSLCipher = null; + $openSSLTag = null; + $openSSLIV = null; + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $fileSize, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + $file = $file + ->setAttribute('$permissions', $permissions) + ->setAttribute('signature', $fileHash) + ->setAttribute('mimeType', $mimeType) + ->setAttribute('sizeActual', $sizeActual) + ->setAttribute('algorithm', $algorithm) + ->setAttribute('openSSLVersion', $openSSLVersion) + ->setAttribute('openSSLCipher', $openSSLCipher) + ->setAttribute('openSSLTag', $openSSLTag) + ->setAttribute('openSSLIV', $openSSLIV) + ->setAttribute('metadata', $metadata) + ->setAttribute('chunksUploaded', $chunksUploaded); + + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } + } else { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + $file = $file + ->setAttribute('chunksUploaded', $chunksUploaded) + ->setAttribute('metadata', $metadata); + + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + try { + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } + } + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket); + + $metadata = null; // was causing leaks as it was passed by reference + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($file, Response::MODEL_FILE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php new file mode 100644 index 0000000000..09e019335a --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php @@ -0,0 +1,17 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId') + ->desc('Get file') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFile', + description: '/docs/references/storage/get-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('mode') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + Response $response, + Database $dbForProject, + string $mode + ) { + $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $response->dynamic($file, Response::MODEL_FILE); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php new file mode 100644 index 0000000000..5f33f9f323 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php @@ -0,0 +1,17 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files') + ->desc('List files') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'listFiles', + description: '/docs/references/storage/list-files.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE_LIST, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('queries', [], new Files(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Files::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('mode') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + array $queries, + string $search, + bool $includeTotal, + Response $response, + Database $dbForProject, + string $mode + ) { + $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(\Utopia\Database\Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $queries = Query::parseQueries($queries); + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $fileId = $cursor->getValue(); + + if ($fileSecurity && !$valid) { + $cursorDocument = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $cursorDocument = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "File '{$fileId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + + try { + if ($fileSecurity && !$valid) { + $files = $dbForProject->find('bucket_' . $bucket->getSequence(), $queries); + $total = $includeTotal ? $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT) : 0; + } else { + $files = Authorization::skip(fn() => $dbForProject->find('bucket_' . $bucket->getSequence(), $queries)); + $total = $includeTotal ? Authorization::skip(fn() => $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT)) : 0; + } + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } catch (OrderException $e) { + throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + $response->dynamic(new Document([ + 'files' => $files, + 'total' => $total, + ]), Response::MODEL_FILE_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php new file mode 100644 index 0000000000..dd14feef6e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Get.php @@ -0,0 +1,65 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Get bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'getBucket', + description: '/docs/references/storage/get-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + Response $response, + Database $dbForProject + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $response->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php new file mode 100644 index 0000000000..9f83479671 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php @@ -0,0 +1,131 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/storage/buckets/:bucketId') + ->desc('Update bucket') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].update') + ->label('audits.event', 'bucket.update') + ->label('audits.resource', 'bucket/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'updateBucket', + description: '/docs/references/storage/update-bucket.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('name', null, new Text(128), 'Bucket name', false) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) + ->param('maximumFileSize', fn(array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn(array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) + ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) + ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) + ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $name, + ?array $permissions, + bool $fileSecurity, + bool $enabled, + ?int $maximumFileSize, + array $allowedFileExtensions, + ?string $compression, + ?bool $encryption, + bool $antivirus, + bool $transformations, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $permissions ??= $bucket->getPermissions(); + $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int) System::getEnv('_APP_STORAGE_LIMIT', 0)); + $allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []); + $enabled ??= $bucket->getAttribute('enabled', true); + $encryption ??= $bucket->getAttribute('encryption', true); + $antivirus ??= $bucket->getAttribute('antivirus', true); + $compression ??= $bucket->getAttribute('compression', Compression::NONE); + $transformations ??= $bucket->getAttribute('transformations', true); + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions); + + $bucket = $dbForProject->updateDocument('buckets', $bucket->getId(), $bucket + ->setAttribute('name', $name) + ->setAttribute('$permissions', $permissions) + ->setAttribute('maximumFileSize', $maximumFileSize) + ->setAttribute('allowedFileExtensions', $allowedFileExtensions) + ->setAttribute('fileSecurity', $fileSecurity) + ->setAttribute('enabled', $enabled) + ->setAttribute('encryption', $encryption) + ->setAttribute('compression', $compression) + ->setAttribute('antivirus', $antivirus) + ->setAttribute('transformations', $transformations)); + + $dbForProject->updateCollection('bucket_' . $bucket->getSequence(), $permissions, $fileSecurity); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()); + + $response->dynamic($bucket, Response::MODEL_BUCKET); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php new file mode 100644 index 0000000000..74f12852be --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/XList.php @@ -0,0 +1,117 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets') + ->desc('List buckets') + ->groups(['api', 'storage']) + ->label('scope', 'buckets.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'buckets', + name: 'listBuckets', + description: '/docs/references/storage/list-buckets.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_BUCKET_LIST, + ) + ] + )) + ->param('queries', [], new Buckets(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Buckets::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action( + array $queries, + string $search, + bool $includeTotal, + Response $response, + Database $dbForProject + ) { + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $bucketId = $cursor->getValue(); + $cursorDocument = $dbForProject->getDocument('buckets', $bucketId); + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Bucket '{$bucketId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + try { + $buckets = $dbForProject->find('buckets', $queries); + $total = $includeTotal ? $dbForProject->count('buckets', $filterQueries, APP_LIMIT_COUNT) : 0; + } catch (OrderException $e) { + throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + $response->dynamic(new Document([ + 'buckets' => $buckets, + 'total' => $total, + ]), Response::MODEL_BUCKET_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php new file mode 100644 index 0000000000..496ac54582 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php @@ -0,0 +1,17 @@ +addService('http', new Http()); + } +} diff --git a/src/Appwrite/Platform/Modules/Storage/Services/Http.php b/src/Appwrite/Platform/Modules/Storage/Services/Http.php new file mode 100644 index 0000000000..e60571eff0 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Storage/Services/Http.php @@ -0,0 +1,51 @@ +type = Service::TYPE_HTTP; + + // Buckets + $this->addAction(CreateBucket::getName(), new CreateBucket()); + $this->addAction(GetBucket::getName(), new GetBucket()); + $this->addAction(ListBuckets::getName(), new ListBuckets()); + $this->addAction(UpdateBucket::getName(), new UpdateBucket()); + $this->addAction(DeleteBucket::getName(), new DeleteBucket()); + + // Files + $this->addAction(CreateFile::getName(), new CreateFile()); + $this->addAction(GetFile::getName(), new GetFile()); + $this->addAction(ListFiles::getName(), new ListFiles()); + $this->addAction(UpdateFile::getName(), new UpdateFile()); + $this->addAction(DeleteFile::getName(), new DeleteFile()); + $this->addAction(GetFilePreview::getName(), new GetFilePreview()); + $this->addAction(GetFileDownload::getName(), new GetFileDownload()); + $this->addAction(GetFileView::getName(), new GetFileView()); + $this->addAction(GetFileForPush::getName(), new GetFileForPush()); + + // Usage + $this->addAction(ListUsage::getName(), new ListUsage()); + $this->addAction(GetBucketUsage::getName(), new GetBucketUsage()); + } +} From f4f4ad9c7dfc95d31483a280d3f1b3b2599a2657 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 29 Dec 2025 01:51:07 +0000 Subject: [PATCH 15/25] format --- .../Modules/Storage/Http/Buckets/Create.php | 3 +-- .../Storage/Http/Buckets/Files/Create.php | 7 +++---- .../Storage/Http/Buckets/Files/Get.php | 4 ++-- .../Storage/Http/Buckets/Files/Push/Get.php | 2 +- .../Storage/Http/Buckets/Files/XList.php | 8 ++++---- .../Modules/Storage/Http/Buckets/Update.php | 2 +- .../Modules/Storage/Services/Http.php | 20 +++++++++---------- 7 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php index a4d28ab487..00daef061e 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Create.php @@ -5,7 +5,6 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; -use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Validator\CustomId; @@ -67,7 +66,7 @@ class Create extends Action ->param('permissions', null, new Nullable(new \Utopia\Database\Validator\Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn(array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn(array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 8640dedb8b..b2d9af5a08 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files; -use Ahc\Jwt\JWT; use Appwrite\ClamAV\Network; use Appwrite\Event\Event; use Appwrite\Extend\Exception; @@ -108,7 +107,7 @@ class Create extends Action Device $deviceForFiles, Device $deviceForLocal ) { - $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = User::isApp(Authorization::getRoles()); $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); @@ -384,7 +383,7 @@ class Create extends Action if (!$validator->isValid($bucket->getCreate())) { throw new Exception(Exception::USER_UNAUTHORIZED); } - $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); } } else { if ($file->isEmpty()) { @@ -431,7 +430,7 @@ class Create extends Action } try { - $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); } catch (NotFoundException) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php index 102ea4d34e..e19fa8ae88 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php @@ -60,7 +60,7 @@ class Get extends Action Database $dbForProject, string $mode ) { - $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = User::isApp(Authorization::getRoles()); $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); @@ -79,7 +79,7 @@ class Get extends Action if ($fileSecurity && !$valid) { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); } else { - $file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); } if ($file->isEmpty()) { diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php index 182cc7e172..b70ada75d3 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php @@ -11,7 +11,7 @@ class Get extends Action return 'getFileForPush'; } - // FILE PUSH - GET /v1/storage/buckets/:bucketId/files/:fileId/push + // FILE PUSH - GET /v1/storage/buckets/:bucketId/files/:fileId/push // Endpoint implementation from /app/controllers/api/storage.php lines 1487-1641 // Provides file access for push notifications with JWT validation } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php index bdd8f4b493..f9448f7d87 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php @@ -73,7 +73,7 @@ class XList extends Action Database $dbForProject, string $mode ) { - $bucket = Authorization::skip(fn() => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = User::isApp(Authorization::getRoles()); $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); @@ -115,7 +115,7 @@ class XList extends Action if ($fileSecurity && !$valid) { $cursorDocument = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); } else { - $cursorDocument = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); } if ($cursorDocument->isEmpty()) { @@ -132,8 +132,8 @@ class XList extends Action $files = $dbForProject->find('bucket_' . $bucket->getSequence(), $queries); $total = $includeTotal ? $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT) : 0; } else { - $files = Authorization::skip(fn() => $dbForProject->find('bucket_' . $bucket->getSequence(), $queries)); - $total = $includeTotal ? Authorization::skip(fn() => $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT)) : 0; + $files = Authorization::skip(fn () => $dbForProject->find('bucket_' . $bucket->getSequence(), $queries)); + $total = $includeTotal ? Authorization::skip(fn () => $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT)) : 0; } } catch (NotFoundException) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php index 9f83479671..44f0192fb4 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Update.php @@ -63,7 +63,7 @@ class Update extends Action ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn(array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn(array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) diff --git a/src/Appwrite/Platform/Modules/Storage/Services/Http.php b/src/Appwrite/Platform/Modules/Storage/Services/Http.php index e60571eff0..95fe160f8b 100644 --- a/src/Appwrite/Platform/Modules/Storage/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Storage/Services/Http.php @@ -4,20 +4,20 @@ namespace Appwrite\Platform\Modules\Storage\Services; use Appwrite\Platform\Modules\Storage\Http\Buckets\Create as CreateBucket; use Appwrite\Platform\Modules\Storage\Http\Buckets\Delete as DeleteBucket; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Create as CreateFile; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Delete as DeleteFile; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Download\Get as GetFileDownload; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Get as GetFile; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Preview\Get as GetFilePreview; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Push\Get as GetFileForPush; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Update as UpdateFile; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\View\Get as GetFileView; +use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\XList as ListFiles; use Appwrite\Platform\Modules\Storage\Http\Buckets\Get as GetBucket; use Appwrite\Platform\Modules\Storage\Http\Buckets\Update as UpdateBucket; use Appwrite\Platform\Modules\Storage\Http\Buckets\XList as ListBuckets; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Create as CreateFile; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Delete as DeleteFile; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Get as GetFile; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Preview\Get as GetFilePreview; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Download\Get as GetFileDownload; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\View\Get as GetFileView; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Push\Get as GetFileForPush; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Update as UpdateFile; -use Appwrite\Platform\Modules\Storage\Http\Buckets\Files\XList as ListFiles; -use Appwrite\Platform\Modules\Storage\Http\Usage\XList as ListUsage; use Appwrite\Platform\Modules\Storage\Http\Usage\Get as GetBucketUsage; +use Appwrite\Platform\Modules\Storage\Http\Usage\XList as ListUsage; use Utopia\Platform\Service; class Http extends Service From f99cb20d05f7e63be7abc56d7553a78f18af24b6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 29 Dec 2025 07:44:03 +0000 Subject: [PATCH 16/25] Initialize storage module and remove storage and fix remaining endpoints --- src/Appwrite/Platform/Appwrite.php | 2 + .../Storage/Http/Buckets/Files/Delete.php | 101 ++++++- .../Http/Buckets/Files/Download/Get.php | 202 ++++++++++++- .../Http/Buckets/Files/Preview/Get.php | 276 +++++++++++++++++- .../Storage/Http/Buckets/Files/Push/Get.php | 196 ++++++++++++- .../Storage/Http/Buckets/Files/Update.php | 104 ++++++- .../Storage/Http/Buckets/Files/View/Get.php | 214 +++++++++++++- .../Modules/Storage/Http/Usage/Get.php | 126 +++++++- .../Modules/Storage/Http/Usage/XList.php | 109 ++++++- 9 files changed, 1306 insertions(+), 24 deletions(-) diff --git a/src/Appwrite/Platform/Appwrite.php b/src/Appwrite/Platform/Appwrite.php index 4aa135c4f1..a34c79308a 100644 --- a/src/Appwrite/Platform/Appwrite.php +++ b/src/Appwrite/Platform/Appwrite.php @@ -10,6 +10,7 @@ use Appwrite\Platform\Modules\Functions; use Appwrite\Platform\Modules\Projects; use Appwrite\Platform\Modules\Proxy; use Appwrite\Platform\Modules\Sites; +use Appwrite\Platform\Modules\Storage; use Appwrite\Platform\Modules\Tokens; use Utopia\Platform\Platform; @@ -26,5 +27,6 @@ class Appwrite extends Platform $this->addModule(new Console\Module()); $this->addModule(new Proxy\Module()); $this->addModule(new Tokens\Module()); + $this->addModule(new Storage\Module()); } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php index 09e019335a..a7ad0851d7 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php @@ -2,16 +2,111 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files; +use Appwrite\Event\Delete as DeleteEvent; +use Appwrite\Event\Event; +use Appwrite\Extend\Exception; +use Appwrite\SDK\AuthType; +use Appwrite\SDK\ContentType; +use Appwrite\SDK\Method; +use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Response; +use Utopia\Database\Database; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\UID; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; class Delete extends Action { + use HTTP; + public static function getName() { return 'deleteFile'; } - // FILE DELETE - DELETE /v1/storage/buckets/:bucketId/files/:fileId - // Endpoint implementation from /app/controllers/api/storage.php lines 1758-1864 - // Deletes file from storage device and database with proper cleanup + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId') + ->desc('Delete file') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].files.[fileId].delete') + ->label('audits.event', 'file.delete') + ->label('audits.resource', 'file/{request.fileId}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'deleteFile', + description: '/docs/references/storage/delete-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_NOCONTENT, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('fileId', '', new UID(), 'File ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + Response $response, + Database $dbForProject, + DeleteEvent $queueForDeletes, + Event $queueForEvents + ) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + // Validate delete permission + $validator = new Authorization(Database::PERMISSION_DELETE); + $validBucketDelete = $validator->isValid($bucket->getDelete()); + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + + if (!$validBucketDelete && !$fileSecurity) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + // Fetch file based on security + if ($fileSecurity && !$validBucketDelete) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + if (!$dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB'); + } + + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($file); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setPayload($response->output($file, Response::MODEL_FILE)); + + $response->noContent(); + } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php index 3efc003fe8..45e3b83375 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Download/Get.php @@ -2,16 +2,212 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Download; +use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; +use Appwrite\SDK\AuthType; +use Appwrite\SDK\ContentType; +use Appwrite\SDK\Method; +use Appwrite\SDK\MethodType; +use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Documents\User; +use Appwrite\Utopia\Response; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\UID; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; +use Utopia\Storage\Compression\Compression; +use Utopia\Storage\Device; +use Utopia\Swoole\Request; +use Utopia\System\System; +use Utopia\Validator\Text; class Get extends Action { + use HTTP; + public static function getName() { return 'getFileDownload'; } - // FILE DOWNLOAD - GET /v1/storage/buckets/:bucketId/files/:fileId/download - // Endpoint implementation from /app/controllers/api/storage.php lines 1154-1314 - // Provides file download with range request support and proper headers + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/download') + ->desc('Get file for download') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFileDownload', + description: '/docs/references/storage/get-file-download.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::ANY, + type: MethodType::LOCATION + )) + ->param('bucketId', '', new UID(), 'Storage bucket ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. + ->param('token', '', new Text(512), 'File token for accessing this file.', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('mode') + ->inject('resourceToken') + ->inject('deviceForFiles') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + ?string $token, + Request $request, + Response $response, + Database $dbForProject, + string $mode, + Document $resourceToken, + Device $deviceForFiles + ) { + /* @type Document $bucket */ + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid && !$isToken) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid && !$isToken) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + /* @type Document $file */ + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { + $end = min(($start + MAX_OUTPUT_CHUNK_SIZE - 1), ($size - 1)); + } + + if ($unit !== 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $response + ->setContentType($file->getAttribute('mimeType')) + ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days + ->addHeader('X-Peak', \memory_get_peak_usage()) + ->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"') + ; + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $deviceForFiles->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + return; + } + $response->send($source); + return; + } + + if (!empty($rangeHeader)) { + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); + return; + } + + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForFiles->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForFiles->read($path)); + } + } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php index 5f33f9f323..9c4e49d0bb 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php @@ -2,16 +2,286 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Preview; +use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; +use Appwrite\SDK\AuthType; +use Appwrite\SDK\ContentType; +use Appwrite\SDK\Method; +use Appwrite\SDK\MethodType; +use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Documents\User; +use Appwrite\Utopia\Response; +use Utopia\CLI\Console; +use Utopia\Config\Config; +use Utopia\Database\Database; +use Utopia\Database\DateTime; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\UID; +use Utopia\Image\Image; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; +use Utopia\Storage\Compression\Compression; +use Utopia\Storage\Device; +use Utopia\Swoole\Request; +use Utopia\System\System; +use Utopia\Validator\HexColor; +use Utopia\Validator\Range; +use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; class Get extends Action { + use HTTP; + public static function getName() { return 'getFilePreview'; } - // FILE PREVIEW - GET /v1/storage/buckets/:bucketId/files/:fileId/preview - // Endpoint implementation from /app/controllers/api/storage.php lines 938-1153 - // Provides image preview generation with crop, transformation, and rendering capabilities + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/preview') + ->desc('Get file preview') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('cache', true) + ->label('cache.resourceType', 'bucket/{request.bucketId}') + ->label('cache.resource', 'file/{request.fileId}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFilePreview', + description: '/docs/references/storage/get-file-preview.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_NONE + ) + ], + type: MethodType::LOCATION, + contentType: ContentType::IMAGE + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID') + ->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true) + ->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true) + ->param('gravity', Image::GRAVITY_CENTER, new WhiteList(Image::getGravityTypes()), 'Image crop gravity. Can be one of ' . implode(",", Image::getGravityTypes()), true) + ->param('quality', -1, new Range(-1, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to keep existing image quality.', true) + ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) + ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) + ->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true) + ->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) + ->param('rotation', 0, new Range(-360, 360), 'Preview image rotation in degrees. Pass an integer between -360 and 360.', true) + ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) + ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) + // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. + ->param('token', '', new Text(512), 'File token for accessing this file.', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('resourceToken') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->inject('project') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + int $width, + int $height, + string $gravity, + int $quality, + int $borderWidth, + string $borderColor, + int $borderRadius, + float $opacity, + int $rotation, + string $background, + string $output, + ?string $token, + Request $request, + Response $response, + Database $dbForProject, + Document $resourceToken, + Device $deviceForFiles, + Device $deviceForLocal, + Document $project + ) { + + if (!\extension_loaded('imagick')) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); + } + + /* @type Document $bucket */ + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!$bucket->getAttribute('transformations', true) && !$isAPIKey && !$isPrivilegedUser) { + throw new Exception(Exception::STORAGE_BUCKET_TRANSFORMATIONS_DISABLED); + } + + $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid && !$isToken) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid && !$isToken) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + /* @type Document $file */ + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $inputs = Config::getParam('storage-inputs'); + $outputs = Config::getParam('storage-outputs'); + $fileLogos = Config::getParam('storage-logos'); + + $path = $file->getAttribute('path'); + $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); + $algorithm = $file->getAttribute('algorithm', Compression::NONE); + $cipher = $file->getAttribute('openSSLCipher'); + $mime = $file->getAttribute('mimeType'); + if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) System::getEnv('_APP_STORAGE_PREVIEW_LIMIT', APP_STORAGE_READ_BUFFER)) { + if (!\in_array($mime, $inputs)) { + $path = (\array_key_exists($mime, $fileLogos)) ? $fileLogos[$mime] : $fileLogos['default']; + } else { + // it was an image but the file size exceeded the limit + $path = $fileLogos['default_image']; + } + + $algorithm = Compression::NONE; + $cipher = null; + $background = (empty($background)) ? 'eceff1' : $background; + $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); + $deviceForFiles = $deviceForLocal; + } + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + if (empty($output)) { + // when file extension is provided but it's not one of our + // supported outputs we fallback to `jpg` + if (!empty($type) && !array_key_exists($type, $outputs)) { + $type = 'jpg'; + } + + // when file extension is not provided and the mime type is not one of our supported outputs + // we fallback to `jpg` output format + $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; + } + + $startTime = \microtime(true); + + $source = $deviceForFiles->read($path); + + $downloadTime = \microtime(true) - $startTime; + + if (!empty($cipher)) { // Decrypt + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + $decryptionTime = \microtime(true) - $startTime - $downloadTime; + + switch ($algorithm) { + case Compression::ZSTD: + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + $decompressionTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime; + + try { + $image = new Image($source); + } catch (\Exception $e) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, $e->getMessage()); + } + + $image->crop((int) $width, (int) $height, $gravity); + + if (!empty($opacity) || $opacity === 0) { + $image->setOpacity($opacity); + } + + if (!empty($background)) { + $image->setBackground('#' . $background); + } + + if (!empty($borderWidth)) { + $image->setBorder($borderWidth, '#' . $borderColor); + } + + if (!empty($borderRadius)) { + $image->setBorderRadius($borderRadius); + } + + if (!empty($rotation)) { + $image->setRotation(($rotation + 360) % 360); + } + + $data = $image->output($output, $quality); + + $renderingTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime - $decompressionTime; + + $totalTime = \microtime(true) - $startTime; + + Console::info("File preview rendered,project=" . $project->getId() . ",bucket=" . $bucketId . ",file=" . $file->getId() . ",uri=" . $request->getURI() . ",total=" . $totalTime . ",rendering=" . $renderingTime . ",decryption=" . $decryptionTime . ",decompression=" . $decompressionTime . ",download=" . $downloadTime); + + $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; + + //Do not update transformedAt if it's a console user + if (!User::isPrivileged(Authorization::getRoles())) { + $transformedAt = $file->getAttribute('transformedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { + $file->setAttribute('transformedAt', DateTime::now()); + Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + } + } + + $response + ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days + ->setContentType($contentType) + ->file($data); + + unset($image); + } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php index b70ada75d3..67372435b1 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Push/Get.php @@ -2,16 +2,206 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files\Push; +use Ahc\Jwt\JWT; +use Ahc\Jwt\JWTException; +use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; +use Appwrite\Utopia\Database\Documents\User; +use Appwrite\Utopia\Response; +use Utopia\Config\Config; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\UID; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; +use Utopia\Storage\Compression\Compression; +use Utopia\Storage\Device; +use Utopia\Swoole\Request; +use Utopia\System\System; +use Utopia\Validator\Text; class Get extends Action { + use HTTP; + public static function getName() { return 'getFileForPush'; } - // FILE PUSH - GET /v1/storage/buckets/:bucketId/files/:fileId/push - // Endpoint implementation from /app/controllers/api/storage.php lines 1487-1641 - // Provides file access for push notifications with JWT validation + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/push') + ->desc('Get file for push notification') + ->groups(['api', 'storage']) + ->label('scope', 'public') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->param('jwt', '', new Text(2048, 0), 'JSON Web Token to validate', true) + ->inject('response') + ->inject('request') + ->inject('dbForProject') + ->inject('dbForPlatform') + ->inject('project') + ->inject('mode') + ->inject('deviceForFiles') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + string $jwt, + Response $response, + Request $request, + Database $dbForProject, + Database $dbForPlatform, + Document $project, + string $mode, + Device $deviceForFiles + ) { + $decoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); + + try { + $decoded = $decoder->decode($jwt); + } catch (JWTException) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ( + $decoded['projectId'] !== $project->getId() || + $decoded['bucketId'] !== $bucketId || + $decoded['fileId'] !== $fileId + ) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + $isInternal = $decoded['internal'] ?? false; + $disposition = $decoded['disposition'] ?? 'inline'; + $dbForProject = $isInternal ? $dbForPlatform : $dbForProject; + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $mimes = Config::getParam('storage-mimes'); + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + $contentType = 'text/plain'; + + if (\in_array($file->getAttribute('mimeType'), $mimes)) { + $contentType = $file->getAttribute('mimeType'); + } + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { + $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); + } + + if ($unit != 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', "bytes $start-$end/$size") + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $response + ->setContentType($contentType) + ->addHeader('Content-Security-Policy', 'script-src none;') + ->addHeader('X-Content-Type-Options', 'nosniff') + ->addHeader('Content-Disposition', $disposition . '; filename="' . $file->getAttribute('name', '') . '"') + ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days + ->addHeader('X-Peak', \memory_get_peak_usage()); + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $deviceForFiles->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + return; + } + $response->send($source); + return; + } + + if (!empty($rangeHeader)) { + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); + return; + } + + $size = $deviceForFiles->getFileSize($path); + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForFiles->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForFiles->read($path)); + } + } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php index 91d6783bc5..f961bab184 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php @@ -2,16 +2,114 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files; +use Appwrite\Event\Event; +use Appwrite\Extend\Exception; +use Appwrite\SDK\AuthType; +use Appwrite\SDK\Method; +use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Response; +use Utopia\Database\Database; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\Permissions; +use Utopia\Database\Validator\UID; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; +use Utopia\Validator\Nullable; +use Utopia\Validator\Text; class Update extends Action { + use HTTP; + public static function getName() { return 'updateFile'; } - // FILE UPDATE - PUT /v1/storage/buckets/:bucketId/files/:fileId - // Endpoint implementation from /app/controllers/api/storage.php lines 1642-1757 - // Updates file metadata like name and permissions + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId') + ->desc('Update file') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('event', 'buckets.[bucketId].files.[fileId].update') + ->label('audits.event', 'file.update') + ->label('audits.resource', 'file/{response.$id}') + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'updateFile', + description: '/docs/references/storage/update-file.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FILE, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('fileId', '', new UID(), 'File ID.') + ->param('name', null, new Text(128), 'File name.', true) + ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + ?string $name, + ?array $permissions, + Response $response, + Database $dbForProject, + Event $queueForEvents + ) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + + $bucketUpdateValidator = new Authorization(Database::PERMISSION_UPDATE); + $bucketUpdateValid = $bucketUpdateValidator->isValid($bucket->getUpdate()); + + if (!$bucketUpdateValid && !$fileSecurity) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + // Fetch file depending on fileSecurity & bucket permission + if ($fileSecurity && !$bucketUpdateValid) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + // Aggregate provided permissions with existing ones if null + $permissions = Permission::aggregate($permissions ?? $file->getPermissions()); + + $name ??= $file->getAttribute('name'); + + $file = $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file + ->setAttribute('name', $name) + ->setAttribute('$permissions', $permissions)); + + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()); + + $response->dynamic($file, Response::MODEL_FILE); + } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php index 2339efd93b..41ee95b165 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/View/Get.php @@ -2,16 +2,224 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files\View; +use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; +use Appwrite\SDK\AuthType; +use Appwrite\SDK\ContentType; +use Appwrite\SDK\Method; +use Appwrite\SDK\MethodType; +use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Documents\User; +use Appwrite\Utopia\Response; +use Utopia\Config\Config; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\UID; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; +use Utopia\Storage\Compression\Compression; +use Utopia\Storage\Device; +use Utopia\Swoole\Request; +use Utopia\System\System; +use Utopia\Validator\Text; class Get extends Action { + use HTTP; + public static function getName() { return 'getFileView'; } - // FILE VIEW - GET /v1/storage/buckets/:bucketId/files/:fileId/view - // Endpoint implementation from /app/controllers/api/storage.php lines 1315-1486 - // Provides file view inline with content type enforcement and security headers + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/buckets/:bucketId/files/:fileId/view') + ->desc('Get file for view') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: 'files', + name: 'getFileView', + description: '/docs/references/storage/get-file-view.md', + auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_NONE, + ) + ], + contentType: ContentType::ANY, + type: MethodType::LOCATION + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. + ->param('token', '', new Text(512), 'File token for accessing this file.', true) + ->inject('response') + ->inject('request') + ->inject('dbForProject') + ->inject('mode') + ->inject('resourceToken') + ->inject('deviceForFiles') + ->callback($this->action(...)); + } + + public function action( + string $bucketId, + string $fileId, + ?string $token, + Response $response, + Request $request, + Database $dbForProject, + string $mode, + Document $resourceToken, + Device $deviceForFiles + ) { + /* @type Document $bucket */ + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $fileSecurity = $bucket->getAttribute('fileSecurity', false); + $validator = new Authorization(Database::PERMISSION_READ); + $valid = $validator->isValid($bucket->getRead()); + if (!$fileSecurity && !$valid && !$isToken) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($fileSecurity && !$valid && !$isToken) { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + /* @type Document $file */ + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + + if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $mimes = Config::getParam('storage-mimes'); + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + $contentType = 'text/plain'; + + if (\in_array($file->getAttribute('mimeType'), $mimes)) { + $contentType = $file->getAttribute('mimeType'); + } + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { + $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); + } + + if ($unit != 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', "bytes $start-$end/$size") + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $response + ->setContentType($contentType) + ->addHeader('Content-Security-Policy', 'script-src none;') + ->addHeader('X-Content-Type-Options', 'nosniff') + ->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"') + ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days + ->addHeader('X-Peak', \memory_get_peak_usage()) + ; + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $deviceForFiles->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + if (!empty($source)) { + if (!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + return; + } + $response->send($source); + return; + } + + if (!empty($rangeHeader)) { + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); + return; + } + + $size = $deviceForFiles->getFileSize($path); + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForFiles->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForFiles->read($path)); + } + } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php index 496ac54582..b816e83f72 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Usage/Get.php @@ -2,16 +2,136 @@ namespace Appwrite\Platform\Modules\Storage\Http\Usage; +use Appwrite\Extend\Exception; +use Appwrite\SDK\AuthType; +use Appwrite\SDK\Method; +use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Response; +use Utopia\Config\Config; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\UID; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; +use Utopia\Validator\WhiteList; class Get extends Action { + use HTTP; + public static function getName() { return 'getBucketUsage'; } - // BUCKET USAGE - GET /v1/storage/:bucketId/usage - // Endpoint implementation from /app/controllers/api/storage.php lines 1952-2053 - // Returns bucket-specific usage statistics + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/:bucketId/usage') + ->desc('Get bucket usage stats') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: null, + name: 'getBucketUsage', + description: '/docs/references/storage/get-bucket-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_USAGE_BUCKETS, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Bucket ID.') + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('project') + ->inject('dbForProject') + ->inject('getLogsDB') + ->callback($this->action(...)); + } + + public function action(string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, callable $getLogsDB) + { + $dbForLogs = call_user_func($getLogsDB, $project); + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES), + str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_STORAGE), + str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED), + ]; + + Authorization::skip(function () use ($dbForProject, $dbForLogs, $bucket, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $db = ($metric === str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED)) + ? $dbForLogs + : $dbForProject; + + $result = $db->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; + $period = $days['period']; + $results = $db->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + + $format = match ($days['period']) { + '1h' => 'Y-m-d\\TH:00:00.000P', + '1d' => 'Y-m-d\\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['data'] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric]['data'][] = [ + 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; + } + } + + $response->dynamic(new Document([ + 'range' => $range, + 'filesTotal' => $usage[$metrics[0]]['total'], + 'filesStorageTotal' => $usage[$metrics[1]]['total'], + 'files' => $usage[$metrics[0]]['data'], + 'storage' => $usage[$metrics[1]]['data'], + 'imageTransformations' => $usage[$metrics[2]]['data'], + 'imageTransformationsTotal' => $usage[$metrics[2]]['total'], + ]), Response::MODEL_USAGE_BUCKETS); + } } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php index 6d4bc921ef..d29fa7c1b4 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Usage/XList.php @@ -2,16 +2,119 @@ namespace Appwrite\Platform\Modules\Storage\Http\Usage; +use Appwrite\SDK\AuthType; +use Appwrite\SDK\Method; +use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Response; +use Utopia\Config\Config; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; +use Utopia\Platform\Scope\HTTP; +use Utopia\Validator\WhiteList; class XList extends Action { + use HTTP; + public static function getName() { return 'getUsage'; } - // STORAGE USAGE - GET /v1/storage/usage - // Endpoint implementation from /app/controllers/api/storage.php lines 1865-1951 - // Returns global storage usage statistics + public function __construct() + { + $this + ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/storage/usage') + ->desc('Get storage usage stats') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('resourceType', RESOURCE_TYPE_BUCKETS) + ->label('sdk', new Method( + namespace: 'storage', + group: null, + name: 'getUsage', + description: '/docs/references/storage/get-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_USAGE_STORAGE, + ) + ] + )) + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } + + public function action(string $range, Response $response, Database $dbForProject) + { + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_BUCKETS, + METRIC_FILES, + METRIC_FILES_STORAGE, + ]; + + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $result = $dbForProject->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + $format = match ($days['period']) { + '1h' => 'Y-m-d\\TH:00:00.000P', + '1d' => 'Y-m-d\\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['data'] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric]['data'][] = [ + 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; + } + } + + $response->dynamic(new Document([ + 'range' => $range, + 'bucketsTotal' => $usage[$metrics[0]]['total'], + 'filesTotal' => $usage[$metrics[1]]['total'], + 'filesStorageTotal' => $usage[$metrics[2]]['total'], + 'buckets' => $usage[$metrics[0]]['data'], + 'files' => $usage[$metrics[1]]['data'], + 'storage' => $usage[$metrics[2]]['data'], + ]), Response::MODEL_USAGE_STORAGE); + } } From 8cf12f685ab6622e3e175f016a7ab6042174f8c8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 29 Dec 2025 07:44:08 +0000 Subject: [PATCH 17/25] remove controller --- app/controllers/api/storage.php | 2052 ------------------------------- 1 file changed, 2052 deletions(-) delete mode 100644 app/controllers/api/storage.php diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php deleted file mode 100644 index ec4cc25ea3..0000000000 --- a/app/controllers/api/storage.php +++ /dev/null @@ -1,2052 +0,0 @@ -desc('Create bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].create') - ->label('audits.event', 'bucket.create') - ->label('audits.resource', 'bucket/{response.$id}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'createBucket', - description: '/docs/references/storage/create-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_CREATED, - model: Response::MODEL_BUCKET, - ) - ] - )) - ->param('bucketId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') - ->param('name', '', new Text(128), 'Bucket name') - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) - ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) - ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) - ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) - ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) - ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) - ->inject('response') - ->inject('dbForProject') - ->inject('queueForEvents') - ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, ?string $compression, ?bool $encryption, bool $antivirus, bool $transformations, Response $response, Database $dbForProject, Event $queueForEvents) { - - $bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId; - - // Map aggregate permissions into the multiple permissions they represent. - $permissions = Permission::aggregate($permissions) ?? []; - $compression ??= Compression::NONE; - $encryption ??= true; - try { - $files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? []; - if (empty($files)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Files collection is not configured.'); - } - - $attributes = []; - $indexes = []; - - foreach ($files['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => $attribute['$id'], - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } - - foreach ($files['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => $index['$id'], - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } - - $dbForProject->createDocument('buckets', new Document([ - '$id' => $bucketId, - '$collection' => 'buckets', - '$permissions' => $permissions, - 'name' => $name, - 'maximumFileSize' => $maximumFileSize, - 'allowedFileExtensions' => $allowedFileExtensions, - 'fileSecurity' => $fileSecurity, - 'enabled' => $enabled, - 'compression' => $compression, - 'encryption' => $encryption, - 'antivirus' => $antivirus, - 'transformations' => $transformations, - 'search' => implode(' ', [$bucketId, $name]), - ])); - - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - $dbForProject->createCollection('bucket_' . $bucket->getSequence(), $attributes, $indexes, permissions: $permissions, documentSecurity: $fileSecurity); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_BUCKET_ALREADY_EXISTS); - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ; - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($bucket, Response::MODEL_BUCKET); - }); - -App::get('/v1/storage/buckets') - ->desc('List buckets') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'listBuckets', - description: '/docs/references/storage/list-buckets.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUCKET_LIST, - ) - ] - )) - ->param('queries', [], new Buckets(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Buckets::ALLOWED_ATTRIBUTES), true) - ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) - ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) - ->inject('response') - ->inject('dbForProject') - ->action(function (array $queries, string $search, bool $includeTotal, Response $response, Database $dbForProject) { - - try { - $queries = Query::parseQueries($queries); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); - } - - if (!empty($search)) { - $queries[] = Query::search('search', $search); - } - - /** - * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries - */ - $cursor = \array_filter($queries, function ($query) { - return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); - }); - $cursor = reset($cursor); - if ($cursor) { - /** @var Query $cursor */ - - $validator = new Cursor(); - if (!$validator->isValid($cursor)) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); - } - - $bucketId = $cursor->getValue(); - $cursorDocument = $dbForProject->getDocument('buckets', $bucketId); - - if ($cursorDocument->isEmpty()) { - throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Bucket '{$bucketId}' for the 'cursor' value not found."); - } - - $cursor->setValue($cursorDocument); - } - - $filterQueries = Query::groupByType($queries)['filters']; - try { - $buckets = $dbForProject->find('buckets', $queries); - $total = $includeTotal ? $dbForProject->count('buckets', $filterQueries, APP_LIMIT_COUNT) : 0; - } catch (OrderException $e) { - throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); - } - $response->dynamic(new Document([ - 'buckets' => $buckets, - 'total' => $total, - ]), Response::MODEL_BUCKET_LIST); - }); - -App::get('/v1/storage/buckets/:bucketId') - ->desc('Get bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'getBucket', - description: '/docs/references/storage/get-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUCKET, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->inject('response') - ->inject('dbForProject') - ->action(function (string $bucketId, Response $response, Database $dbForProject) { - - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $response->dynamic($bucket, Response::MODEL_BUCKET); - }); - -App::put('/v1/storage/buckets/:bucketId') - ->desc('Update bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].update') - ->label('audits.event', 'bucket.update') - ->label('audits.resource', 'bucket/{response.$id}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'updateBucket', - description: '/docs/references/storage/update-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUCKET, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->param('name', null, new Text(128), 'Bucket name', false) - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) - ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) - ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) - ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) - ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) - ->param('transformations', true, new Boolean(true), 'Are image transformations enabled?', true) - ->inject('response') - ->inject('dbForProject') - ->inject('queueForEvents') - ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, ?string $compression, ?bool $encryption, bool $antivirus, bool $transformations, Response $response, Database $dbForProject, Event $queueForEvents) { - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $permissions ??= $bucket->getPermissions(); - $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int) System::getEnv('_APP_STORAGE_LIMIT', 0)); - $allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []); - $enabled ??= $bucket->getAttribute('enabled', true); - $encryption ??= $bucket->getAttribute('encryption', true); - $antivirus ??= $bucket->getAttribute('antivirus', true); - $compression ??= $bucket->getAttribute('compression', Compression::NONE); - $transformations ??= $bucket->getAttribute('transformations', true); - - // Map aggregate permissions into the multiple permissions they represent. - $permissions = Permission::aggregate($permissions); - - $bucket = $dbForProject->updateDocument('buckets', $bucket->getId(), $bucket - ->setAttribute('name', $name) - ->setAttribute('$permissions', $permissions) - ->setAttribute('maximumFileSize', $maximumFileSize) - ->setAttribute('allowedFileExtensions', $allowedFileExtensions) - ->setAttribute('fileSecurity', $fileSecurity) - ->setAttribute('enabled', $enabled) - ->setAttribute('encryption', $encryption) - ->setAttribute('compression', $compression) - ->setAttribute('antivirus', $antivirus) - ->setAttribute('transformations', $transformations)); - - $dbForProject->updateCollection('bucket_' . $bucket->getSequence(), $permissions, $fileSecurity); - - $queueForEvents - ->setParam('bucketId', $bucket->getId()); - - $response->dynamic($bucket, Response::MODEL_BUCKET); - }); - -App::delete('/v1/storage/buckets/:bucketId') - ->desc('Delete bucket') - ->groups(['api', 'storage']) - ->label('scope', 'buckets.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('audits.event', 'bucket.delete') - ->label('event', 'buckets.[bucketId].delete') - ->label('audits.resource', 'bucket/{request.bucketId}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'buckets', - name: 'deleteBucket', - description: '/docs/references/storage/delete-bucket.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_NOCONTENT, - model: Response::MODEL_NONE, - ) - ], - contentType: ContentType::NONE - )) - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->inject('response') - ->inject('dbForProject') - ->inject('queueForDeletes') - ->inject('queueForEvents') - ->action(function (string $bucketId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents) { - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - if (!$dbForProject->deleteDocument('buckets', $bucketId)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove bucket from DB'); - } - - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($bucket); - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setPayload($response->output($bucket, Response::MODEL_BUCKET)) - ; - - $response->noContent(); - }); - -App::post('/v1/storage/buckets/:bucketId/files') - ->alias('/v1/storage/files') - ->desc('Create file') - ->groups(['api', 'storage']) - ->label('scope', 'files.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('audits.event', 'file.create') - ->label('event', 'buckets.[bucketId].files.[fileId].create') - ->label('audits.resource', 'file/{response.$id}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'createFile', - description: '/docs/references/storage/create-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_CREATED, - model: Response::MODEL_FILE, - ) - ], - type: MethodType::UPLOAD, - requestType: ContentType::MULTIPART - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new CustomId(), 'File ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') - ->param('file', [], new File(), 'Binary file. Appwrite SDKs provide helpers to handle file input. [Learn about file input](https://appwrite.io/docs/products/storage/upload-download#input-file).', skipValidation: true) - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE])), 'An array of permission strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->inject('request') - ->inject('response') - ->inject('dbForProject') - ->inject('user') - ->inject('queueForEvents') - ->inject('mode') - ->inject('deviceForFiles') - ->inject('deviceForLocal') - ->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode, Device $deviceForFiles, Device $deviceForLocal) { - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $allowedPermissions = [ - Database::PERMISSION_READ, - Database::PERMISSION_UPDATE, - Database::PERMISSION_DELETE, - ]; - - // Map aggregate permissions to into the set of individual permissions they represent. - $permissions = Permission::aggregate($permissions, $allowedPermissions); - - // Add permissions for current the user if none were provided. - if (\is_null($permissions)) { - $permissions = []; - if (!empty($user->getId()) && !$isPrivilegedUser) { - foreach ($allowedPermissions as $permission) { - $permissions[] = (new Permission($permission, 'user', $user->getId()))->toString(); - } - } - } - - // Users can only manage their own roles, API keys and Admin users can manage any - $roles = Authorization::getRoles(); - if (!$isAPIKey && !$isPrivilegedUser) { - foreach (Database::PERMISSIONS as $type) { - foreach ($permissions as $permission) { - $permission = Permission::parse($permission); - if ($permission->getPermission() != $type) { - continue; - } - $role = (new Role( - $permission->getRole(), - $permission->getIdentifier(), - $permission->getDimension() - ))->toString(); - if (!Authorization::isRole($role)) { - throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); - } - } - } - } - - $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); - if ($maximumFileSize > (int) System::getEnv('_APP_STORAGE_LIMIT', 0)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Maximum bucket file size is larger than _APP_STORAGE_LIMIT'); - } - - - $file = $request->getFiles('file'); - - // GraphQL multipart spec adds files with index keys - if (empty($file)) { - $file = $request->getFiles(0); - } - - if (empty($file)) { - throw new Exception(Exception::STORAGE_FILE_EMPTY); - } - - // Make sure we handle a single file and multiple files the same way - $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - - $contentRange = $request->getHeader('content-range'); - $fileId = $fileId === 'unique()' ? ID::unique() : $fileId; - $chunk = 1; - $chunks = 1; - - if (!empty($contentRange)) { - $start = $request->getContentRangeStart(); - $end = $request->getContentRangeEnd(); - $fileSize = $request->getContentRangeSize(); - $fileId = $request->getHeader('x-appwrite-id', $fileId); - // TODO make `end >= $fileSize` in next breaking version - if (is_null($start) || is_null($end) || is_null($fileSize) || $end > $fileSize) { - throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE); - } - - $idValidator = new UID(); - if (!$idValidator->isValid($fileId)) { - throw new Exception(Exception::STORAGE_INVALID_APPWRITE_ID); - } - - // TODO remove the condition that checks `$end === $fileSize` in next breaking version - if ($end === $fileSize - 1 || $end === $fileSize) { - //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk - $chunks = $chunk = -1; - } else { - // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) - $chunks = (int) ceil($fileSize / ($end + 1 - $start)); - $chunk = (int) ($start / ($end + 1 - $start)) + 1; - } - } - - /** - * Validators - */ - // Check if file type is allowed - $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); - $fileExt = new FileExt($allowedFileExtensions); - if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { - throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, 'File extension not allowed'); - } - - // Check if file size is exceeding allowed limit - $fileSizeValidator = new FileSize($maximumFileSize); - if (!$fileSizeValidator->isValid($fileSize)) { - throw new Exception(Exception::STORAGE_INVALID_FILE_SIZE, 'File size not allowed'); - } - - $upload = new Upload(); - if (!$upload->isValid($fileTmpName)) { - throw new Exception(Exception::STORAGE_INVALID_FILE); - } - - // Save to storage - $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); - $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); - $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root - - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - - $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; - if (!$file->isEmpty()) { - $chunks = $file->getAttribute('chunksTotal', 1); - $uploaded = $file->getAttribute('chunksUploaded', 0); - $metadata = $file->getAttribute('metadata', []); - - if ($chunk === -1) { - $chunk = $chunks; - } - - if ($uploaded === $chunks) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } - } - - $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); - - if (empty($chunksUploaded)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); - } - - if ($chunksUploaded === $chunks) { - if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { - $antivirus = new Network( - System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), - (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) - ); - - if (!$antivirus->fileScan($path)) { - $deviceForFiles->delete($path); - throw new Exception(Exception::STORAGE_INVALID_FILE); - } - } - - $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption - $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption - $data = ''; - // Compression - $algorithm = $bucket->getAttribute('compression', Compression::NONE); - if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { - $data = $deviceForFiles->read($path); - switch ($algorithm) { - case Compression::ZSTD: - $compressor = new Zstd(); - break; - case Compression::GZIP: - default: - $compressor = new GZIP(); - break; - } - $data = $compressor->compress($data); - } else { - // reset the algorithm to none as we do not compress the file - // if file size exceedes the APP_STORAGE_READ_BUFFER - // regardless the bucket compression algoorithm - $algorithm = Compression::NONE; - } - - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - if (empty($data)) { - $data = $deviceForFiles->read($path); - } - $key = System::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); - } - - if (!empty($data)) { - if (!$deviceForFiles->write($path, $data, $mimeType)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); - } - } - - $sizeActual = $deviceForFiles->getFileSize($path); - - $openSSLVersion = null; - $openSSLCipher = null; - $openSSLTag = null; - $openSSLIV = null; - - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - $openSSLVersion = '1'; - $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; - $openSSLTag = \bin2hex($tag); - $openSSLIV = \bin2hex($iv); - } - - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => $fileId, - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => $fileHash, - 'mimeType' => $mimeType, - 'sizeOriginal' => $fileSize, - 'sizeActual' => $sizeActual, - 'algorithm' => $algorithm, - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'openSSLVersion' => $openSSLVersion, - 'openSSLCipher' => $openSSLCipher, - 'openSSLTag' => $openSSLTag, - 'openSSLIV' => $openSSLIV, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - $file = $file - ->setAttribute('$permissions', $permissions) - ->setAttribute('signature', $fileHash) - ->setAttribute('mimeType', $mimeType) - ->setAttribute('sizeActual', $sizeActual) - ->setAttribute('algorithm', $algorithm) - ->setAttribute('openSSLVersion', $openSSLVersion) - ->setAttribute('openSSLCipher', $openSSLCipher) - ->setAttribute('openSSLTag', $openSSLTag) - ->setAttribute('openSSLIV', $openSSLIV) - ->setAttribute('metadata', $metadata) - ->setAttribute('chunksUploaded', $chunksUploaded); - - /** - * Validate create permission and skip authorization in updateDocument - * Without this, the file creation will fail when user doesn't have update permission - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we validate create permission instead of update - */ - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } - } else { - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => ID::custom($fileId), - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => '', - 'mimeType' => '', - 'sizeOriginal' => $fileSize, - 'sizeActual' => 0, - 'algorithm' => '', - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - $file = $file - ->setAttribute('chunksUploaded', $chunksUploaded) - ->setAttribute('metadata', $metadata); - - /** - * Validate create permission and skip authorization in updateDocument - * Without this, the file creation will fail when user doesn't have update permission - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we validate create permission instead of update - */ - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - try { - $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket); - - $metadata = null; // was causing leaks as it was passed by reference - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($file, Response::MODEL_FILE); - }); - -App::get('/v1/storage/buckets/:bucketId/files') - ->alias('/v1/storage/files') - ->desc('List files') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'listFiles', - description: '/docs/references/storage/list-files.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_FILE_LIST, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('queries', [], new Files(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Files::ALLOWED_ATTRIBUTES), true) - ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) - ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) - ->inject('response') - ->inject('dbForProject') - ->inject('mode') - ->action(function (string $bucketId, array $queries, string $search, bool $includeTotal, Response $response, Database $dbForProject, string $mode) { - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $queries = Query::parseQueries($queries); - - if (!empty($search)) { - $queries[] = Query::search('search', $search); - } - - /** - * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries - */ - $cursor = \array_filter($queries, function ($query) { - return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); - }); - $cursor = reset($cursor); - if ($cursor) { - /** @var Query $cursor */ - - $validator = new Cursor(); - if (!$validator->isValid($cursor)) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); - } - - $fileId = $cursor->getValue(); - - if ($fileSecurity && !$valid) { - $cursorDocument = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if ($cursorDocument->isEmpty()) { - throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "File '{$fileId}' for the 'cursor' value not found."); - } - - $cursor->setValue($cursorDocument); - } - - $filterQueries = Query::groupByType($queries)['filters']; - - try { - if ($fileSecurity && !$valid) { - $files = $dbForProject->find('bucket_' . $bucket->getSequence(), $queries); - $total = $includeTotal ? $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT) : 0; - } else { - $files = Authorization::skip(fn () => $dbForProject->find('bucket_' . $bucket->getSequence(), $queries)); - $total = $includeTotal ? Authorization::skip(fn () => $dbForProject->count('bucket_' . $bucket->getSequence(), $filterQueries, APP_LIMIT_COUNT)) : 0; - } - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } catch (OrderException $e) { - throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); - } - - $response->dynamic(new Document([ - 'files' => $files, - 'total' => $total, - ]), Response::MODEL_FILE_LIST); - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId') - ->alias('/v1/storage/files/:fileId') - ->desc('Get file') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFile', - description: '/docs/references/storage/get-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_FILE, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - ->inject('response') - ->inject('dbForProject') - ->inject('mode') - ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, string $mode) { - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $response->dynamic($file, Response::MODEL_FILE); - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') - ->alias('/v1/storage/files/:fileId/preview') - ->desc('Get file preview') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('cache', true) - ->label('cache.resourceType', 'bucket/{request.bucketId}') - ->label('cache.resource', 'file/{request.fileId}') - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFilePreview', - description: '/docs/references/storage/get-file-preview.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_NONE - ) - ], - type: MethodType::LOCATION, - contentType: ContentType::IMAGE - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID') - ->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true) - ->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true) - ->param('gravity', Image::GRAVITY_CENTER, new WhiteList(Image::getGravityTypes()), 'Image crop gravity. Can be one of ' . implode(",", Image::getGravityTypes()), true) - ->param('quality', -1, new Range(-1, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to keep existing image quality.', true) - ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) - ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) - ->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true) - ->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) - ->param('rotation', 0, new Range(-360, 360), 'Preview image rotation in degrees. Pass an integer between -360 and 360.', true) - ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) - ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) - // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. - ->param('token', '', new Text(512), 'File token for accessing this file.', true) - ->inject('request') - ->inject('response') - ->inject('dbForProject') - ->inject('resourceToken') - ->inject('deviceForFiles') - ->inject('deviceForLocal') - ->inject('project') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, ?string $token, Request $request, Response $response, Database $dbForProject, Document $resourceToken, Device $deviceForFiles, Device $deviceForLocal, Document $project) { - - if (!\extension_loaded('imagick')) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); - } - - /* @type Document $bucket */ - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - if (!$bucket->getAttribute('transformations', true) && !$isAPIKey && !$isPrivilegedUser) { - throw new Exception(Exception::STORAGE_BUCKET_TRANSFORMATIONS_DISABLED); - } - - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid && !$isToken) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid && !$isToken) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - /* @type Document $file */ - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $inputs = Config::getParam('storage-inputs'); - $outputs = Config::getParam('storage-outputs'); - $fileLogos = Config::getParam('storage-logos'); - - $path = $file->getAttribute('path'); - $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $algorithm = $file->getAttribute('algorithm', Compression::NONE); - $cipher = $file->getAttribute('openSSLCipher'); - $mime = $file->getAttribute('mimeType'); - if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) System::getEnv('_APP_STORAGE_PREVIEW_LIMIT', APP_STORAGE_READ_BUFFER)) { - if (!\in_array($mime, $inputs)) { - $path = (\array_key_exists($mime, $fileLogos)) ? $fileLogos[$mime] : $fileLogos['default']; - } else { - // it was an image but the file size exceeded the limit - $path = $fileLogos['default_image']; - } - - $algorithm = Compression::NONE; - $cipher = null; - $background = (empty($background)) ? 'eceff1' : $background; - $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $deviceForFiles = $deviceForLocal; - } - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - if (empty($output)) { - // when file extension is provided but it's not one of our - // supported outputs we fallback to `jpg` - if (!empty($type) && !array_key_exists($type, $outputs)) { - $type = 'jpg'; - } - - // when file extension is not provided and the mime type is not one of our supported outputs - // we fallback to `jpg` output format - $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; - } - - $startTime = \microtime(true); - - $source = $deviceForFiles->read($path); - - $downloadTime = \microtime(true) - $startTime; - - if (!empty($cipher)) { // Decrypt - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - $decryptionTime = \microtime(true) - $startTime - $downloadTime; - - switch ($algorithm) { - case Compression::ZSTD: - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - $decompressionTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime; - - try { - $image = new Image($source); - } catch (ImagickException $e) { - throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, $e->getMessage()); - } - - $image->crop((int) $width, (int) $height, $gravity); - - if (!empty($opacity) || $opacity === 0) { - $image->setOpacity($opacity); - } - - if (!empty($background)) { - $image->setBackground('#' . $background); - } - - if (!empty($borderWidth)) { - $image->setBorder($borderWidth, '#' . $borderColor); - } - - if (!empty($borderRadius)) { - $image->setBorderRadius($borderRadius); - } - - if (!empty($rotation)) { - $image->setRotation(($rotation + 360) % 360); - } - - $data = $image->output($output, $quality); - - $renderingTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime - $decompressionTime; - - $totalTime = \microtime(true) - $startTime; - - Console::info("File preview rendered,project=" . $project->getId() . ",bucket=" . $bucketId . ",file=" . $file->getId() . ",uri=" . $request->getURI() . ",total=" . $totalTime . ",rendering=" . $renderingTime . ",decryption=" . $decryptionTime . ",decompression=" . $decompressionTime . ",download=" . $downloadTime); - - $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - - //Do not update transformedAt if it's a console user - if (!User::isPrivileged(Authorization::getRoles())) { - $transformedAt = $file->getAttribute('transformedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { - $file->setAttribute('transformedAt', DateTime::now()); - Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); - } - } - - $response - ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days - ->setContentType($contentType) - ->file($data); - - unset($image); - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') - ->alias('/v1/storage/files/:fileId/download') - ->desc('Get file for download') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFileDownload', - description: '/docs/references/storage/get-file-download.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_NONE - ) - ], - type: MethodType::LOCATION, - contentType: ContentType::ANY, - )) - ->param('bucketId', '', new UID(), 'Storage bucket ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. - ->param('token', '', new Text(512), 'File token for accessing this file.', true) - ->inject('request') - ->inject('response') - ->inject('dbForProject') - ->inject('mode') - ->inject('resourceToken') - ->inject('deviceForFiles') - ->action(function (string $bucketId, string $fileId, ?string $token, Request $request, Response $response, Database $dbForProject, string $mode, Document $resourceToken, Device $deviceForFiles) { - /* @type Document $bucket */ - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid && !$isToken) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid && !$isToken) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - /* @type Document $file */ - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $path = $file->getAttribute('path', ''); - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); - } - - $size = $file->getAttribute('sizeOriginal', 0); - - $rangeHeader = $request->getHeader('range'); - if (!empty($rangeHeader)) { - $start = $request->getRangeStart(); - $end = $request->getRangeEnd(); - $unit = $request->getRangeUnit(); - - if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { - $end = min(($start + MAX_OUTPUT_CHUNK_SIZE - 1), ($size - 1)); - } - - if ($unit !== 'bytes' || $start >= $end || $end >= $size) { - throw new Exception(Exception::STORAGE_INVALID_RANGE); - } - - $response - ->addHeader('Accept-Ranges', 'bytes') - ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) - ->addHeader('Content-Length', $end - $start + 1) - ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - } - - $response - ->setContentType($file->getAttribute('mimeType')) - ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days - ->addHeader('X-Peak', \memory_get_peak_usage()) - ->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"') - ; - - $source = ''; - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceForFiles->read($path); - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - if (!empty($source)) { - if (!empty($rangeHeader)) { - $response->send(substr($source, $start, ($end - $start + 1))); - return; - } - $response->send($source); - return; - } - - if (!empty($rangeHeader)) { - $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); - return; - } - - if ($size > APP_STORAGE_READ_BUFFER) { - for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { - $response->chunk( - $deviceForFiles->read( - $path, - ($i * MAX_OUTPUT_CHUNK_SIZE), - min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) - ), - (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size - ); - } - } else { - $response->send($deviceForFiles->read($path)); - } - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') - ->alias('/v1/storage/files/:fileId/view') - ->desc('Get file for view') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'getFileView', - description: '/docs/references/storage/get-file-view.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_NONE, - ) - ], - type: MethodType::LOCATION, - contentType: ContentType::ANY, - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. - ->param('token', '', new Text(512), 'File token for accessing this file.', true) - ->inject('response') - ->inject('request') - ->inject('dbForProject') - ->inject('mode') - ->inject('resourceToken') - ->inject('deviceForFiles') - ->action(function (string $bucketId, string $fileId, ?string $token, Response $response, Request $request, Database $dbForProject, string $mode, Document $resourceToken, Device $deviceForFiles) { - /* @type Document $bucket */ - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_READ); - $valid = $validator->isValid($bucket->getRead()); - if (!$fileSecurity && !$valid && !$isToken) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($fileSecurity && !$valid && !$isToken) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - /* @type Document $file */ - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $mimes = Config::getParam('storage-mimes'); - - $path = $file->getAttribute('path', ''); - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); - } - - $contentType = 'text/plain'; - - if (\in_array($file->getAttribute('mimeType'), $mimes)) { - $contentType = $file->getAttribute('mimeType'); - } - - $size = $file->getAttribute('sizeOriginal', 0); - - $rangeHeader = $request->getHeader('range'); - if (!empty($rangeHeader)) { - $start = $request->getRangeStart(); - $end = $request->getRangeEnd(); - $unit = $request->getRangeUnit(); - - if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { - $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); - } - - if ($unit != 'bytes' || $start >= $end || $end >= $size) { - throw new Exception(Exception::STORAGE_INVALID_RANGE); - } - - $response - ->addHeader('Accept-Ranges', 'bytes') - ->addHeader('Content-Range', "bytes $start-$end/$size") - ->addHeader('Content-Length', $end - $start + 1) - ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - } - - $response - ->setContentType($contentType) - ->addHeader('Content-Security-Policy', 'script-src none;') - ->addHeader('X-Content-Type-Options', 'nosniff') - ->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"') - ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days - ->addHeader('X-Peak', \memory_get_peak_usage()) - ; - - $source = ''; - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceForFiles->read($path); - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - if (!empty($source)) { - if (!empty($rangeHeader)) { - $response->send(substr($source, $start, ($end - $start + 1))); - return; - } - $response->send($source); - return; - } - - if (!empty($rangeHeader)) { - $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); - return; - } - - $size = $deviceForFiles->getFileSize($path); - if ($size > APP_STORAGE_READ_BUFFER) { - for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { - $response->chunk( - $deviceForFiles->read( - $path, - ($i * MAX_OUTPUT_CHUNK_SIZE), - min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) - ), - (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size - ); - } - } else { - $response->send($deviceForFiles->read($path)); - } - }); - -App::get('/v1/storage/buckets/:bucketId/files/:fileId/push') - ->desc('Get file for push notification') - ->groups(['api', 'storage']) - ->label('scope', 'public') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - ->param('jwt', '', new Text(2048, 0), 'JSON Web Token to validate', true) - ->inject('response') - ->inject('request') - ->inject('dbForProject') - ->inject('dbForPlatform') - ->inject('project') - ->inject('mode') - ->inject('deviceForFiles') - ->action(function (string $bucketId, string $fileId, string $jwt, Response $response, Request $request, Database $dbForProject, Database $dbForPlatform, Document $project, string $mode, Device $deviceForFiles) { - $decoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); - - try { - $decoded = $decoder->decode($jwt); - } catch (JWTException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - if ( - $decoded['projectId'] !== $project->getId() || - $decoded['bucketId'] !== $bucketId || - $decoded['fileId'] !== $fileId - ) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $isInternal = $decoded['internal'] ?? false; - $disposition = $decoded['disposition'] ?? 'inline'; - $dbForProject = $isInternal ? $dbForPlatform : $dbForProject; - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - $mimes = Config::getParam('storage-mimes'); - - $path = $file->getAttribute('path', ''); - - if (!$deviceForFiles->exists($path)) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); - } - - $contentType = 'text/plain'; - - if (\in_array($file->getAttribute('mimeType'), $mimes)) { - $contentType = $file->getAttribute('mimeType'); - } - - $size = $file->getAttribute('sizeOriginal', 0); - - $rangeHeader = $request->getHeader('range'); - if (!empty($rangeHeader)) { - $start = $request->getRangeStart(); - $end = $request->getRangeEnd(); - $unit = $request->getRangeUnit(); - - if ($end === null || $end - $start > APP_STORAGE_READ_BUFFER) { - $end = min(($start + APP_STORAGE_READ_BUFFER - 1), ($size - 1)); - } - - if ($unit != 'bytes' || $start >= $end || $end >= $size) { - throw new Exception(Exception::STORAGE_INVALID_RANGE); - } - - $response - ->addHeader('Accept-Ranges', 'bytes') - ->addHeader('Content-Range', "bytes $start-$end/$size") - ->addHeader('Content-Length', $end - $start + 1) - ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - } - - $response - ->setContentType($contentType) - ->addHeader('Content-Security-Policy', 'script-src none;') - ->addHeader('X-Content-Type-Options', 'nosniff') - ->addHeader('Content-Disposition', $disposition . '; filename="' . $file->getAttribute('name', '') . '"') - ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days - ->addHeader('X-Peak', \memory_get_peak_usage()); - - $source = ''; - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceForFiles->read($path); - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; - } - - if (!empty($source)) { - if (!empty($rangeHeader)) { - $response->send(substr($source, $start, ($end - $start + 1))); - return; - } - $response->send($source); - return; - } - - if (!empty($rangeHeader)) { - $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); - return; - } - - $size = $deviceForFiles->getFileSize($path); - if ($size > APP_STORAGE_READ_BUFFER) { - for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { - $response->chunk( - $deviceForFiles->read( - $path, - ($i * MAX_OUTPUT_CHUNK_SIZE), - min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) - ), - (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size - ); - } - } else { - $response->send($deviceForFiles->read($path)); - } - }); - -App::put('/v1/storage/buckets/:bucketId/files/:fileId') - ->alias('/v1/storage/files/:fileId') - ->desc('Update file') - ->groups(['api', 'storage']) - ->label('scope', 'files.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].files.[fileId].update') - ->label('audits.event', 'file.update') - ->label('audits.resource', 'file/{response.$id}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'updateFile', - description: '/docs/references/storage/update-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_FILE, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File unique ID.') - ->param('name', null, new Nullable(new Text(255)), 'Name of the file', true) - ->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE])), 'An array of permission string. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->inject('response') - ->inject('dbForProject') - ->inject('user') - ->inject('mode') - ->inject('queueForEvents') - ->action(function (string $bucketId, string $fileId, ?string $name, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $queueForEvents) { - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_UPDATE); - $valid = $validator->isValid($bucket->getUpdate()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - // Read permission should not be required for update - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - // Map aggregate permissions into the multiple permissions they represent. - $permissions = Permission::aggregate($permissions, [ - Database::PERMISSION_READ, - Database::PERMISSION_UPDATE, - Database::PERMISSION_DELETE, - ]); - - // Users can only manage their own roles, API keys and Admin users can manage any - $roles = Authorization::getRoles(); - if (!User::isApp($roles) && !User::isPrivileged($roles) && !\is_null($permissions)) { - foreach (Database::PERMISSIONS as $type) { - foreach ($permissions as $permission) { - $permission = Permission::parse($permission); - if ($permission->getPermission() != $type) { - continue; - } - $role = (new Role( - $permission->getRole(), - $permission->getIdentifier(), - $permission->getDimension() - ))->toString(); - if (!Authorization::isRole($role)) { - throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); - } - } - } - } - - if (\is_null($permissions)) { - $permissions = $file->getPermissions() ?? []; - } - - $file->setAttribute('$permissions', $permissions); - - if (!is_null($name)) { - $file->setAttribute('name', $name); - } - - try { - if ($fileSecurity && !$valid) { - $file = $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file); - } else { - $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket) - ; - - $response->dynamic($file, Response::MODEL_FILE); - }); - -App::delete('/v1/storage/buckets/:bucketId/files/:fileId') - ->desc('Delete file') - ->groups(['api', 'storage']) - ->label('scope', 'files.write') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('event', 'buckets.[bucketId].files.[fileId].delete') - ->label('audits.event', 'file.delete') - ->label('audits.resource', 'file/{request.fileId}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'storage', - group: 'files', - name: 'deleteFile', - description: '/docs/references/storage/delete-file.md', - auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_NOCONTENT, - model: Response::MODEL_NONE, - ) - ], - contentType: ContentType::NONE - )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') - ->param('fileId', '', new UID(), 'File ID.') - ->inject('response') - ->inject('dbForProject') - ->inject('queueForEvents') - ->inject('mode') - ->inject('deviceForFiles') - ->inject('queueForDeletes') - ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Device $deviceForFiles, Delete $queueForDeletes) { - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - $isAPIKey = User::isApp(Authorization::getRoles()); - $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); - - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(Database::PERMISSION_DELETE); - $valid = $validator->isValid($bucket->getDelete()); - if (!$fileSecurity && !$valid) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - // Read permission should not be required for delete - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - - if ($file->isEmpty()) { - throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); - } - - // Make sure we don't delete the file before the document permission check occurs - if ($fileSecurity && !$valid && !$validator->isValid($file->getDelete())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - - $deviceDeleted = false; - if ($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) { - $deviceDeleted = $deviceForFiles->abort( - $file->getAttribute('path'), - ($file->getAttribute('metadata', [])['uploadId'] ?? '') - ); - } else { - $deviceDeleted = $deviceForFiles->delete($file->getAttribute('path')); - } - - if ($deviceDeleted) { - $queueForDeletes - ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) - ->setResourceType('bucket/' . $bucket->getId()) - ->setResource('file/' . $fileId) - ; - - try { - if ($fileSecurity && !$valid) { - $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId)); - } - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - if (!$deleted) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB'); - } - } else { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to delete file from device'); - } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket) - ->setPayload($response->output($file, Response::MODEL_FILE)) - ; - - $response->noContent(); - }); - -/** Storage usage */ -App::get('/v1/storage/usage') - ->desc('Get storage usage stats') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: null, - name: 'getUsage', - description: '/docs/references/storage/get-usage.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_USAGE_STORAGE, - ) - ] - )) - ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('dbForProject') - ->action(function (string $range, Response $response, Database $dbForProject) { - - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - METRIC_BUCKETS, - METRIC_FILES, - METRIC_FILES_STORAGE, - ]; - - $total = []; - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { - foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - - $stats[$metric]['total'] = $result['value'] ?? 0; - $limit = $days['limit']; - $period = $days['period']; - $results = $dbForProject->find('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', [$period]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric]['data'] = []; - foreach ($results as $result) { - $stats[$metric]['data'][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric]['total'] = $stats[$metric]['total']; - $usage[$metric]['data'] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric]['data'][] = [ - 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, - 'date' => $formatDate, - ]; - } - } - $response->dynamic(new Document([ - 'range' => $range, - 'bucketsTotal' => $usage[$metrics[0]]['total'], - 'filesTotal' => $usage[$metrics[1]]['total'], - 'filesStorageTotal' => $usage[$metrics[2]]['total'], - 'buckets' => $usage[$metrics[0]]['data'], - 'files' => $usage[$metrics[1]]['data'], - 'storage' => $usage[$metrics[2]]['data'], - ]), Response::MODEL_USAGE_STORAGE); - }); - -App::get('/v1/storage/:bucketId/usage') - ->desc('Get bucket usage stats') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('resourceType', RESOURCE_TYPE_BUCKETS) - ->label('sdk', new Method( - namespace: 'storage', - group: null, - name: 'getBucketUsage', - description: '/docs/references/storage/get-bucket-usage.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_USAGE_BUCKETS, - ) - ] - )) - ->param('bucketId', '', new UID(), 'Bucket ID.') - ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('project') - ->inject('dbForProject') - ->inject('getLogsDB') - ->action(function (string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, callable $getLogsDB) { - - $dbForLogs = call_user_func($getLogsDB, $project); - $bucket = $dbForProject->getDocument('buckets', $bucketId); - - if ($bucket->isEmpty()) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - - $periods = Config::getParam('usage', []); - $stats = $usage = []; - $days = $periods[$range]; - $metrics = [ - str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES), - str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_STORAGE), - str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED), - ]; - - Authorization::skip(function () use ($dbForProject, $dbForLogs, $bucket, $days, $metrics, &$stats) { - foreach ($metrics as $metric) { - $db = ($metric === str_replace('{bucketInternalId}', $bucket->getSequence(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED)) - ? $dbForLogs - : $dbForProject; - - $result = $db->findOne('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['inf']) - ]); - - $stats[$metric]['total'] = $result['value'] ?? 0; - $limit = $days['limit']; - $period = $days['period']; - $results = $db->find('stats', [ - Query::equal('metric', [$metric]), - Query::equal('period', [$period]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - $stats[$metric]['data'] = []; - foreach ($results as $result) { - $stats[$metric]['data'][$result->getAttribute('time')] = [ - 'value' => $result->getAttribute('value'), - ]; - } - } - }); - - - $format = match ($days['period']) { - '1h' => 'Y-m-d\TH:00:00.000P', - '1d' => 'Y-m-d\T00:00:00.000P', - }; - - foreach ($metrics as $metric) { - $usage[$metric]['total'] = $stats[$metric]['total']; - $usage[$metric]['data'] = []; - $leap = time() - ($days['limit'] * $days['factor']); - while ($leap < time()) { - $leap += $days['factor']; - $formatDate = date($format, $leap); - $usage[$metric]['data'][] = [ - 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, - 'date' => $formatDate, - ]; - } - } - - $response->dynamic(new Document([ - 'range' => $range, - 'filesTotal' => $usage[$metrics[0]]['total'], - 'filesStorageTotal' => $usage[$metrics[1]]['total'], - 'files' => $usage[$metrics[0]]['data'], - 'storage' => $usage[$metrics[1]]['data'], - 'imageTransformations' => $usage[$metrics[2]]['data'], - 'imageTransformationsTotal' => $usage[$metrics[2]]['total'], - ]), Response::MODEL_USAGE_BUCKETS); - }); From 30373980f1c49d69881b92f13936423caf1f3a07 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 29 Dec 2025 07:56:46 +0000 Subject: [PATCH 18/25] fix update endpoint --- .../Storage/Http/Buckets/Files/Update.php | 79 ++++++++++++++----- 1 file changed, 60 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php index f961bab184..be78cc358b 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Update.php @@ -7,9 +7,12 @@ use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; @@ -73,42 +76,80 @@ class Update extends Action ) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - if ($bucket->isEmpty()) { + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } $fileSecurity = $bucket->getAttribute('fileSecurity', false); - - $bucketUpdateValidator = new Authorization(Database::PERMISSION_UPDATE); - $bucketUpdateValid = $bucketUpdateValidator->isValid($bucket->getUpdate()); - - if (!$bucketUpdateValid && !$fileSecurity) { + $validator = new Authorization(Database::PERMISSION_UPDATE); + $valid = $validator->isValid($bucket->getUpdate()); + if (!$fileSecurity && !$valid) { throw new Exception(Exception::USER_UNAUTHORIZED); } - // Fetch file depending on fileSecurity & bucket permission - if ($fileSecurity && !$bucketUpdateValid) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } + // Read permission should not be required for update + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - // Aggregate provided permissions with existing ones if null - $permissions = Permission::aggregate($permissions ?? $file->getPermissions()); + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions, [ + Database::PERMISSION_READ, + Database::PERMISSION_UPDATE, + Database::PERMISSION_DELETE, + ]); - $name ??= $file->getAttribute('name'); + // Users can only manage their own roles, API keys and Admin users can manage any + $roles = Authorization::getRoles(); + if (!User::isApp($roles) && !User::isPrivileged($roles) && !\is_null($permissions)) { + foreach (Database::PERMISSIONS as $type) { + foreach ($permissions as $permission) { + $permission = Permission::parse($permission); + if ($permission->getPermission() != $type) { + continue; + } + $role = (new Role( + $permission->getRole(), + $permission->getIdentifier(), + $permission->getDimension() + ))->toString(); + if (!Authorization::isRole($role)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')'); + } + } + } + } - $file = $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file - ->setAttribute('name', $name) - ->setAttribute('$permissions', $permissions)); + if (\is_null($permissions)) { + $permissions = $file->getPermissions() ?? []; + } + + $file->setAttribute('$permissions', $permissions); + + if (!is_null($name)) { + $file->setAttribute('name', $name); + } + + try { + if ($fileSecurity && !$valid) { + $file = $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file); + } else { + $file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); + } + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } $queueForEvents ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()); + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket) + ; $response->dynamic($file, Response::MODEL_FILE); } From 774e3af61c1f333da8079beb5a5f862f7fac5d9c Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 29 Dec 2025 10:20:32 +0200 Subject: [PATCH 19/25] skip variables subquery --- src/Appwrite/Platform/Workers/Functions.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index d962ddc8a8..8211a46bd5 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -122,14 +122,15 @@ class Functions extends Action $log->addTag('type', $type); if (!empty($events)) { - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while ($sum >= $limit) { $functions = $dbForProject->find('functions', [ + Query::select(['$id', 'events']), // Skip variables subqueries Query::limit($limit), Query::offset($offset), - Query::orderAsc('name'), + Query::orderAsc('$sequence'), ]); $sum = \count($functions); @@ -147,6 +148,11 @@ class Functions extends Action continue; } + /** + * get variables subqueries cached + */ + $function = $dbForProject->getDocument('functions', $function->getId()); + Console::success('Iterating function: ' . $function->getAttribute('name')); $this->execute( From 80bb9a1f335f59e3cb6cd1324e4e5c1760498ede Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 29 Dec 2025 08:41:38 +0000 Subject: [PATCH 20/25] fixe endpoints --- .../Storage/Http/Buckets/Files/Delete.php | 80 ++++++++++++++----- .../Storage/Http/Buckets/Files/Get.php | 4 +- 2 files changed, 60 insertions(+), 24 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php index a7ad0851d7..eccacaafd2 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Delete.php @@ -9,12 +9,15 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\Storage\Device; class Delete extends Action { @@ -37,6 +40,9 @@ class Delete extends Action ->label('event', 'buckets.[bucketId].files.[fileId].delete') ->label('audits.event', 'file.delete') ->label('audits.resource', 'file/{request.fileId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', new Method( namespace: 'storage', group: 'files', @@ -51,12 +57,13 @@ class Delete extends Action ], contentType: ContentType::NONE )) - ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') ->inject('response') ->inject('dbForProject') - ->inject('queueForDeletes') ->inject('queueForEvents') + ->inject('deviceForFiles') + ->inject('queueForDeletes') ->callback($this->action(...)); } @@ -65,47 +72,78 @@ class Delete extends Action string $fileId, Response $response, Database $dbForProject, + Event $queueForEvents, + Device $deviceForFiles, DeleteEvent $queueForDeletes, - Event $queueForEvents ) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - if ($bucket->isEmpty()) { + $isAPIKey = User::isApp(Authorization::getRoles()); + $isPrivilegedUser = User::isPrivileged(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - // Validate delete permission - $validator = new Authorization(Database::PERMISSION_DELETE); - $validBucketDelete = $validator->isValid($bucket->getDelete()); $fileSecurity = $bucket->getAttribute('fileSecurity', false); - - if (!$validBucketDelete && !$fileSecurity) { + $validator = new Authorization(Database::PERMISSION_DELETE); + $valid = $validator->isValid($bucket->getDelete()); + if (!$fileSecurity && !$valid) { throw new Exception(Exception::USER_UNAUTHORIZED); } - // Fetch file based on security - if ($fileSecurity && !$validBucketDelete) { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - } else { - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - } + // Read permission should not be required for delete + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - if (!$dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB'); + // Make sure we don't delete the file before the document permission check occurs + if ($fileSecurity && !$valid && !$validator->isValid($file->getDelete())) { + throw new Exception(Exception::USER_UNAUTHORIZED); } - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($file); + $deviceDeleted = false; + if ($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) { + $deviceDeleted = $deviceForFiles->abort( + $file->getAttribute('path'), + ($file->getAttribute('metadata', [])['uploadId'] ?? '') + ); + } else { + $deviceDeleted = $deviceForFiles->delete($file->getAttribute('path')); + } + + if ($deviceDeleted) { + $queueForDeletes + ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResourceType('bucket/' . $bucket->getId()) + ->setResource('file/' . $fileId) + ; + + try { + if ($fileSecurity && !$valid) { + $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId); + } else { + $deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId)); + } + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!$deleted) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB'); + } + } else { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to delete file from device'); + } $queueForEvents ->setParam('bucketId', $bucket->getId()) ->setParam('fileId', $file->getId()) - ->setPayload($response->output($file, Response::MODEL_FILE)); + ->setContext('bucket', $bucket) + ->setPayload($response->output($file, Response::MODEL_FILE)) + ; $response->noContent(); } diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php index e19fa8ae88..77f163e5fb 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Get.php @@ -49,7 +49,6 @@ class Get extends Action ->param('fileId', '', new UID(), 'File ID.') ->inject('response') ->inject('dbForProject') - ->inject('mode') ->callback($this->action(...)); } @@ -58,7 +57,6 @@ class Get extends Action string $fileId, Response $response, Database $dbForProject, - string $mode ) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -70,7 +68,7 @@ class Get extends Action } $fileSecurity = $bucket->getAttribute('fileSecurity', false); - $validator = new Authorization(\Utopia\Database\Database::PERMISSION_READ); + $validator = new Authorization(Database::PERMISSION_READ); $valid = $validator->isValid($bucket->getRead()); if (!$fileSecurity && !$valid) { throw new Exception(Exception::USER_UNAUTHORIZED); From 1b70bc812b6aa8f101e951430de91aa17cb63c7f Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Mon, 29 Dec 2025 14:51:59 +0530 Subject: [PATCH 21/25] keep certificate renewal in maintenance worker --- .env | 1 - docker-compose.yml | 1 - src/Appwrite/Platform/Tasks/Interval.php | 51 --------------------- src/Appwrite/Platform/Tasks/Maintenance.php | 45 ++++++++++++++++++ 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/.env b/.env index 19ee65350b..88dec63b1c 100644 --- a/.env +++ b/.env @@ -102,7 +102,6 @@ _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_INTERVAL_DOMAIN_VERIFICATION=60 -_APP_INTERVAL_CERTIFICATE_RENEWAL=86400 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/docker-compose.yml b/docker-compose.yml index 4adbd096ab..805be67340 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -823,7 +823,6 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - _APP_INTERVAL_DOMAIN_VERIFICATION - - _APP_INTERVAL_CERTIFICATE_RENEWAL appwrite-task-stats-resources: container_name: appwrite-task-stats-resources diff --git a/src/Appwrite/Platform/Tasks/Interval.php b/src/Appwrite/Platform/Tasks/Interval.php index 74ab9db1f1..2aa15bf7d0 100644 --- a/src/Appwrite/Platform/Tasks/Interval.php +++ b/src/Appwrite/Platform/Tasks/Interval.php @@ -34,19 +34,12 @@ class Interval extends Action Console::success(APP_NAME . ' interval process v1 has started'); $intervalDomainVerification = (int) System::getEnv('_APP_INTERVAL_DOMAIN_VERIFICATION', '60'); // 1 minute - $intervalCertificateRenewal = (int) System::getEnv('_APP_INTERVAL_CERTIFICATE_RENEWAL', '86400'); // 1 day \go(function () use ($dbForPlatform, $queueForCertificates, $intervalDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->verifyDomain($dbForPlatform, $queueForCertificates); }, $intervalDomainVerification); }); - - \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) { - Console::loop(function () use ($dbForPlatform, $queueForCertificates) { - $this->renewCertificates($dbForPlatform, $queueForCertificates); - }, $intervalCertificateRenewal); - }); } private function verifyDomain(Database $dbForPlatform, Certificate $queueForCertificate): void @@ -79,48 +72,4 @@ class Interval extends Action ->trigger(); } } - - private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void - { - $time = DatabaseDateTime::now(); - - $certificates = $dbForPlatform->find('certificates', [ - Query::lessThan('attempts', 5), // Maximum 5 attempts - Query::isNotNull('renewDate'), - Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) - Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) - ]); - - if (\count($certificates) === 0) { - Console::info("[{$time}] No certificates for renewal."); - return; - } - - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - $appRegion = System::getEnv('_APP_REGION', 'default'); - - foreach ($certificates as $certificate) { - $domain = $certificate->getAttribute('domain'); - $rule = $isMd5 ? - $dbForPlatform->getDocument('rules', md5($domain)) : - $dbForPlatform->findOne('rules', [ - Query::equal('domain', [$domain]), - Query::limit(1) - ]); - - if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { - continue; - } - - $queueForCertificate - ->setDomain(new Document([ - 'domain' => $rule->getAttribute('domain'), - 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), - ])) - ->setAction(Certificate::ACTION_GENERATION) - ->trigger(); - } - } } diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 66d3a3d9de..c0914c6544 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -92,6 +92,7 @@ class Maintenance extends Action ->trigger(); $this->notifyDeleteConnections($queueForDeletes); + $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); $this->notifyDeleteCSVExports($queueForDeletes); @@ -113,6 +114,50 @@ class Maintenance extends Action ->trigger(); } + private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void + { + $time = DatabaseDateTime::now(); + + $certificates = $dbForPlatform->find('certificates', [ + Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), + Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) + Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) + ]); + + if (\count($certificates) === 0) { + Console::info("[{$time}] No certificates for renewal."); + return; + } + + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $appRegion = System::getEnv('_APP_REGION', 'default'); + + foreach ($certificates as $certificate) { + $domain = $certificate->getAttribute('domain'); + $rule = $isMd5 ? + $dbForPlatform->getDocument('rules', md5($domain)) : + $dbForPlatform->findOne('rules', [ + Query::equal('domain', [$domain]), + Query::limit(1) + ]); + + if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) { + continue; + } + + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $rule->getAttribute('domain'), + 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), + ])) + ->setAction(Certificate::ACTION_GENERATION) + ->trigger(); + } + } + private function notifyDeleteCache($interval, Delete $queueForDeletes): void { $queueForDeletes From 417bb22790a6eb631a16ab3f59952ec153c77104 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 29 Dec 2025 11:55:21 +0200 Subject: [PATCH 22/25] use Query::contains --- src/Appwrite/Platform/Workers/Functions.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 8211a46bd5..fba5154079 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -128,6 +128,7 @@ class Functions extends Action while ($sum >= $limit) { $functions = $dbForProject->find('functions', [ Query::select(['$id', 'events']), // Skip variables subqueries + Query::contains('events', $events), Query::limit($limit), Query::offset($offset), Query::orderAsc('$sequence'), From 2b96d60c1e8babc8294517693761bf9b7c69af84 Mon Sep 17 00:00:00 2001 From: Hemachandar Date: Mon, 29 Dec 2025 16:01:19 +0530 Subject: [PATCH 23/25] copilot - code quality --- .../Certificates/Exception/CertificateStatus.php | 2 +- src/Appwrite/Certificates/LetsEncrypt.php | 2 +- src/Appwrite/Platform/Tasks/Interval.php | 4 ++-- src/Appwrite/Platform/Workers/Certificates.php | 9 +++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Certificates/Exception/CertificateStatus.php b/src/Appwrite/Certificates/Exception/CertificateStatus.php index 3d94109d0e..ca15a95ed8 100644 --- a/src/Appwrite/Certificates/Exception/CertificateStatus.php +++ b/src/Appwrite/Certificates/Exception/CertificateStatus.php @@ -1,6 +1,6 @@ setDomain(new Document([ 'domain' => $rule->getAttribute('domain'), 'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')), diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 0c4d495724..5132687279 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -116,8 +116,6 @@ class Certificates extends Action default: throw new Exception('Invalid action: ' . $action); } - - } /** @@ -143,7 +141,7 @@ class Certificates extends Action Realtime $queueForRealtime, Certificate $queueForCertificates, Log $log, - ?string $validationDomain = null, + ?string $validationDomain = null ): void { // Get rule $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5' @@ -177,7 +175,6 @@ class Certificates extends Action $this->updateRuleAndSendEvents($rule, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } - // Issue a TLS certificate when domain is verified if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) { $queueForCertificates @@ -317,7 +314,7 @@ class Certificates extends Action $date = \date('H:i:s'); $errorMessage = "\033[90m[{$date}] \033[31mCertificate generation failed: \033[0m\n"; - $attempts = $certificate->getAttribute('attempts', 0) + 1; // // Increase attempts count + $attempts = $certificate->getAttribute('attempts', 0) + 1; // Increase attempts count // Update attributes on certificate document $certificate->setAttributes([ @@ -379,7 +376,7 @@ class Certificates extends Action /** * Update all existing domain documents so they have relation to correct certificate document. - * This solved issues: + * This solves issues: * - when adding a domain for which there is already a certificate * - when renew creates new document? It might? * - overall makes it more reliable From 57157a71b4427b9cdd890cb9b14d875d21e7c626 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 29 Dec 2025 16:44:19 +0530 Subject: [PATCH 24/25] chore: more sdk config flexibility --- src/Appwrite/Platform/Tasks/SDKs.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/SDKs.php b/src/Appwrite/Platform/Tasks/SDKs.php index 859e259b7c..d6e3efa559 100644 --- a/src/Appwrite/Platform/Tasks/SDKs.php +++ b/src/Appwrite/Platform/Tasks/SDKs.php @@ -138,7 +138,7 @@ class SDKs extends Action $target = \realpath(__DIR__ . '/../../../../app') . '/sdks/git/' . $language['key'] . '/'; $readme = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/README.md'); $readme = ($readme) ? \file_get_contents($readme) : ''; - $gettingStarted = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); + $gettingStarted = $language['gettingStarted'] ?? \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); $gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : ''; $examples = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md'); $examples = ($examples) ? \file_get_contents($examples) : ''; @@ -381,7 +381,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ->setName($language['name']) ->setNamespace($language['namespace'] ?? 'appwrite') ->setDescription($language['description'] ?? "Appwrite is an open-source backend as a service server that abstracts and simplifies complex and repetitive development tasks behind a very simple to use REST API. Appwrite aims to help you develop your apps faster and in a more secure way. Use the {$language['name']} SDK to integrate your app with the Appwrite server to easily start interacting with all of Appwrite backend APIs and tools. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)") - ->setShortDescription('Appwrite is an open-source self-hosted backend server that abstracts and simplifies complex and repetitive development tasks behind a very simple REST API') + ->setShortDescription($language['shortDescription'] ?? 'Appwrite is an open-source self-hosted backend server that abstracts and simplifies complex and repetitive development tasks behind a very simple REST API') ->setLicense($license) ->setLicenseContent($licenseContent) ->setVersion($language['version']) From ca877fa71de4bd3601d23986a64a9420ec87137c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 29 Dec 2025 13:24:24 +0000 Subject: [PATCH 25/25] Catch query parse exceptions --- .../Platform/Modules/Storage/Http/Buckets/Files/XList.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php index f9448f7d87..e46fdb2a0a 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/XList.php @@ -89,7 +89,11 @@ class XList extends Action throw new Exception(Exception::USER_UNAUTHORIZED); } - $queries = Query::parseQueries($queries); + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } if (!empty($search)) { $queries[] = Query::search('search', $search);