diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 90ef137fc2..187281716f 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -25446,12 +25446,33 @@ "Temporary Redirect 307", "Permanent Redirect 308" ] + }, + "resourceId": { + "type": "string", + "description": "ID of parent resource.", + "x-example": "" + }, + "resourceType": { + "type": "string", + "description": "Type of parent resource.", + "x-example": "site", + "enum": [ + "site", + "function" + ], + "x-enum-name": "ProxyResourceType", + "x-enum-keys": [ + "Site", + "Function" + ] } }, "required": [ "domain", "url", - "statusCode" + "statusCode", + "resourceId", + "resourceType" ] } } diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index e53a0dfb0b..06a33c9a3d 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -25696,12 +25696,35 @@ "Temporary Redirect 307", "Permanent Redirect 308" ] + }, + "resourceId": { + "type": "string", + "description": "ID of parent resource.", + "default": null, + "x-example": "" + }, + "resourceType": { + "type": "string", + "description": "Type of parent resource.", + "default": null, + "x-example": "site", + "enum": [ + "site", + "function" + ], + "x-enum-name": "ProxyResourceType", + "x-enum-keys": [ + "Site", + "Function" + ] } }, "required": [ "domain", "url", - "statusCode" + "statusCode", + "resourceId", + "resourceType" ] } } diff --git a/app/config/storage/inputs.php b/app/config/storage/inputs.php index edcf667d86..a102dfdcf0 100644 --- a/app/config/storage/inputs.php +++ b/app/config/storage/inputs.php @@ -7,4 +7,5 @@ return [ "png" => "image/png", "heic" => "image/heic", "webp" => "image/webp", + "gif" => "image/gif", ]; diff --git a/app/config/storage/outputs.php b/app/config/storage/outputs.php index 519ff825fe..3e6fd45651 100644 --- a/app/config/storage/outputs.php +++ b/app/config/storage/outputs.php @@ -8,4 +8,5 @@ return [ "webp" => "image/webp", "heic" => "image/heic", "avif" => "image/avif", + "gif" => "image/gif", ]; diff --git a/app/config/storage/resource_limits.php b/app/config/storage/resource_limits.php new file mode 100644 index 0000000000..cfbcea5a47 --- /dev/null +++ b/app/config/storage/resource_limits.php @@ -0,0 +1,6 @@ +setContentType('application/gzip') - ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days + ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') + ->addHeader('Expires', '0') + ->addHeader('Pragma', 'no-cache') ->addHeader('X-Peak', \memory_get_peak_usage()) - ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"'); + ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"'); $size = $device->getFileSize($path); $rangeHeader = $request->getHeader('range'); diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php index d0c9dbbbe3..3396a9c6fd 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php @@ -14,6 +14,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; +use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -65,15 +66,18 @@ class Create extends Action ->param('domain', null, new ValidatorDomain(), 'Domain name.') ->param('url', null, new URL(), 'Target URL of redirection') ->param('statusCode', null, new WhiteList([301, 302, 307, 308]), 'Status code of redirection') + ->param('resourceId', '', new UID(), 'ID of parent resource.') + ->param('resourceType', '', new WhiteList(['site', 'function']), 'Type of parent resource.') ->inject('response') ->inject('project') ->inject('queueForCertificates') ->inject('queueForEvents') ->inject('dbForPlatform') + ->inject('dbForProject') ->callback([$this, 'action']); } - public function action(string $domain, string $url, int $statusCode, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform) + public function action(string $domain, string $url, int $statusCode, string $resourceId, string $resourceType, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject) { $deniedDomains = [ 'localhost', @@ -116,6 +120,15 @@ class Create extends Action throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Domain may not start with http:// or https://.'); } + $collection = match ($resourceType) { + 'site' => 'sites', + 'function' => 'functions' + }; + $resource = $dbForProject->getDocument($collection, $resourceId); + if ($resource->isEmpty()) { + throw new Exception(Exception::RULE_RESOURCE_NOT_FOUND); + } + // TODO: @christyjacob remove once we migrate the rules in 1.7.x $ruleId = System::getEnv('_APP_RULES_FORMAT') === 'md5' ? md5($domain->get()) : ID::unique(); @@ -164,6 +177,9 @@ class Create extends Action 'trigger' => 'manual', 'redirectUrl' => $url, 'redirectStatusCode' => $statusCode, + 'deploymentResourceType' => $resourceType, + 'deploymentResourceId' => $resource->getId(), + 'deploymentResourceInternalId' => $resource->getInternalId(), 'certificateId' => '', 'search' => implode(' ', [$ruleId, $domain->get()]), 'owner' => $owner, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php index e83af0bab6..d3a2f0b814 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php @@ -99,12 +99,14 @@ class Get extends Action } if (!$device->exists($path)) { - throw new Exception(Exception::BUILD_NOT_FOUND); + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } $response ->setContentType('application/gzip') - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache + ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') + ->addHeader('Expires', '0') + ->addHeader('Pragma', 'no-cache') ->addHeader('X-Peak', \memory_get_peak_usage()) ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"'); diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 76309145b8..ce8dbedd73 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,13 +12,13 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Queue\Result\Commit; +use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + protected const int BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,9 +27,7 @@ class Audits extends Action protected function getBatchSize(): int { - return System::getEnv('_APP_ENV', 'development') === 'development' - ? self::BATCH_SIZE_DEVELOPMENT - : self::BATCH_SIZE_PRODUCTION; + return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); } public static function getName(): string @@ -57,13 +55,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return void + * @return Commit|NoCommit * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): void + public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit { $payload = $message->getPayload() ?? []; @@ -123,29 +121,32 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $shouldProcessBatch = \count($this->logs) >= $batchSize; - if (!$shouldProcessBatch && \count($this->logs) > 0) { + $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs['logs']), 0); + $shouldProcessBatch = $logCount >= $batchSize; + if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if ($shouldProcessBatch) { - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); - - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); - - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); - - unset($this->logs[$internalId]); - } - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - } finally { - $this->lastTriggeredTime = time(); - } + if (!$shouldProcessBatch) { + return new NoCommit(); } + + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); + + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); + + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); + + unset($this->logs[$internalId]); + } + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } + $this->lastTriggeredTime = time(); + return new Commit(); } } diff --git a/src/Appwrite/SDK/Specification/Format.php b/src/Appwrite/SDK/Specification/Format.php index 8dbccf5cbb..af3b5c017e 100644 --- a/src/Appwrite/SDK/Specification/Format.php +++ b/src/Appwrite/SDK/Specification/Format.php @@ -113,6 +113,16 @@ abstract class Format protected function getEnumName(string $service, string $method, string $param): ?string { switch ($service) { + case 'proxy': + switch ($method) { + case 'createRedirectRule': + switch ($param) { + case 'resourceType': + return 'ProxyResourceType'; + } + break; + } + break; case 'console': switch ($method) { case 'getResource': @@ -441,7 +451,13 @@ abstract class Format case 'proxy': switch ($method) { case 'createRedirectRule': - return ['Moved Permanently 301', 'Found 302', 'Temporary Redirect 307', 'Permanent Redirect 308']; + switch ($param) { + case 'statusCode': + return ['Moved Permanently 301', 'Found 302', 'Temporary Redirect 307', 'Permanent Redirect 308']; + case 'resourceType': + return ['Site', 'Function']; + } + break; } break; case 'functions': diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index d1d50c2ffd..5f528e660f 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -87,7 +87,7 @@ class Comment $i = 0; foreach ($projects as $projectId => $project) { $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_DOMAIN'); + $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN')); $text .= "## {$project['name']}\n\n"; $text .= "Project ID: `{$projectId}`\n\n"; @@ -201,7 +201,7 @@ class Comment public function generatImage(string $pathLight, string $pathDark, string $alt, int $width): string { $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_DOMAIN'); + $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN')); $imageLight = $protocol . '://' . $hostname . $pathLight; $imageDark = $protocol . '://' . $hostname . $pathDark; diff --git a/tests/e2e/Services/Proxy/ProxyBase.php b/tests/e2e/Services/Proxy/ProxyBase.php index 44e015b751..9f8e92d56f 100644 --- a/tests/e2e/Services/Proxy/ProxyBase.php +++ b/tests/e2e/Services/Proxy/ProxyBase.php @@ -68,7 +68,7 @@ trait ProxyBase return $rule; } - protected function createRedirectRule(string $domain, string $url, int $statusCode): mixed + protected function createRedirectRule(string $domain, string $url, int $statusCode, string $resourceType, string $resourceId): mixed { $rule = $this->client->call(Client::METHOD_POST, '/proxy/rules/redirect', array_merge([ 'content-type' => 'application/json', @@ -77,6 +77,8 @@ trait ProxyBase 'domain' => $domain, 'url' => $url, 'statusCode' => $statusCode, + 'resourceType' => $resourceType, + 'resourceId' => $resourceId, ]); return $rule; @@ -115,9 +117,9 @@ trait ProxyBase return $rule['body']['$id']; } - protected function setupRedirectRule(string $domain, string $url, int $statusCode): string + protected function setupRedirectRule(string $domain, string $url, int $statusCode, string $resourceType, string $resourceId): string { - $rule = $this->createRedirectRule($domain, $url, $statusCode); + $rule = $this->createRedirectRule($domain, $url, $statusCode, $resourceType, $resourceId); $this->assertEquals(201, $rule['headers']['status-code'], 'Failed to setup rule: ' . \json_encode($rule)); diff --git a/tests/e2e/Services/Proxy/ProxyCustomServerTest.php b/tests/e2e/Services/Proxy/ProxyCustomServerTest.php index 9a7ba74ec1..ea310d5449 100644 --- a/tests/e2e/Services/Proxy/ProxyCustomServerTest.php +++ b/tests/e2e/Services/Proxy/ProxyCustomServerTest.php @@ -131,7 +131,9 @@ class ProxyCustomServerTest extends Scope $response = $proxyClient->call(Client::METHOD_GET, '/todos/1'); $this->assertEquals(404, $response['headers']['status-code']); - $ruleId = $this->setupRedirectRule($domain, 'https://jsonplaceholder.typicode.com/todos/1', 301); + $siteId = $this->setupSite()['siteId']; + + $ruleId = $this->setupRedirectRule($domain, 'https://jsonplaceholder.typicode.com/todos/1', 301, 'site', $siteId); $this->assertNotEmpty($ruleId); $response = $proxyClient->call(Client::METHOD_GET, '/todos/1'); @@ -147,7 +149,7 @@ class ProxyCustomServerTest extends Scope $this->assertEquals('https://jsonplaceholder.typicode.com/todos/1', $response['headers']['location']); $domain = \uniqid() . '-redirect-307.custom.localhost'; - $ruleId = $this->setupRedirectRule($domain, 'https://jsonplaceholder.typicode.com/todos/1', 307); + $ruleId = $this->setupRedirectRule($domain, 'https://jsonplaceholder.typicode.com/todos/1', 307, 'site', $siteId); $this->assertNotEmpty($ruleId); $proxyClient = new Client(); @@ -158,6 +160,18 @@ class ProxyCustomServerTest extends Scope $this->assertEquals(307, $response['headers']['status-code']); $this->assertEquals('https://jsonplaceholder.typicode.com/todos/1', $response['headers']['location']); + $rules = $this->listRules([ + 'queries' => [ + Query::equal('type', ['redirect'])->toString(), + Query::equal('trigger', ['manual'])->toString(), + Query::equal('deploymentResourceType', ['site'])->toString(), + Query::equal('deploymentResourceId', [$siteId])->toString(), + ], + ]); + $this->assertEquals(200, $rules['headers']['status-code']); + $this->assertEquals(2, $rules['body']['total']); + + $this->cleanupSite($siteId); $this->cleanupRule($ruleId); }