diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 26c789ce27..98a5b105a3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -18,6 +18,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Buckets; use Appwrite\Utopia\Database\Validator\Queries\Files; use Appwrite\Utopia\Response; use Utopia\App; +use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -953,12 +954,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) // NOTE: this is only for the sdk generator and is not used in the action below and is utilised in `resources.php` for `resourceToken`. ->param('token', '', new Text(512), 'File token for accessing this file.', true) + ->inject('request') ->inject('response') ->inject('dbForProject') ->inject('resourceToken') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, ?string $token, Response $response, Database $dbForProject, Document $resourceToken, Device $deviceForFiles, Device $deviceForLocal) { + ->inject('project') + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, ?string $token, Request $request, Response $response, Database $dbForProject, Document $resourceToken, Device $deviceForFiles, Device $deviceForLocal, Document $project) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1035,8 +1038,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; } + $startTime = \microtime(true); + $source = $deviceForFiles->read($path); + $downloadTime = \microtime(true) - $startTime; + if (!empty($cipher)) { // Decrypt $source = OpenSSL::decrypt( $source, @@ -1048,6 +1055,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ); } + $decryptionTime = \microtime(true) - $startTime - $downloadTime; + switch ($algorithm) { case Compression::ZSTD: $compressor = new Zstd(); @@ -1059,6 +1068,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') break; } + $decompressionTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime; + try { $image = new Image($source); } catch (ImagickException $e) { @@ -1089,6 +1100,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $data = $image->output($output, $quality); + $renderingTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime - $decompressionTime; + + $totalTime = \microtime(true) - $startTime; + + Console::info("File preview rendered,project=" . $project->getId() . ",bucket=" . $bucketId . ",file=" . $file->getId() . ",uri=" . $request->getURI() . ",total=" . $totalTime . ",rendering=" . $renderingTime . ",decryption=" . $decryptionTime . ",decompression=" . $decompressionTime . ",download=" . $downloadTime); + $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; //Do not update transformedAt if it's a console user diff --git a/app/controllers/general.php b/app/controllers/general.php index 44a9cff45b..c18f729c44 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1408,7 +1408,7 @@ App::get('/robots.txt') ->inject('isResourceBlocked') ->inject('previewHostname') ->inject('apiKey') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Log $log, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname, ?Key $apiKey) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname, ?Key $apiKey) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); diff --git a/app/init/registers.php b/app/init/registers.php index 415730f936..3dc0e22dba 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -36,7 +36,8 @@ App::setMode(System::getEnv('_APP_ENV', App::MODE_TYPE_PRODUCTION)); if (!App::isProduction()) { // Allow specific domains to skip public domain validation in dev environment // Useful for existing tests involving webhooks - PublicDomain::allow(['request-catcher']); + PublicDomain::allow(['request-catcher-sms']); + PublicDomain::allow(['request-catcher-webhook']); } $register->set('logger', function () { // Register error logger diff --git a/composer.json b/composer.json index 22ca53ee78..255f82224b 100644 --- a/composer.json +++ b/composer.json @@ -86,7 +86,7 @@ }, "require-dev": { "ext-fileinfo": "*", - "appwrite/sdk-generator": "dev-fix-attribute-creation-errors", + "appwrite/sdk-generator": "0.41.x", "phpunit/phpunit": "9.*", "swoole/ide-helper": "5.1.2", "phpstan/phpstan": "1.8.*", diff --git a/composer.lock b/composer.lock index 45467bd4e5..60d039c562 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "71a256558261b19e11189c5967e5bd07", + "content-hash": "081385d2d7081b323429e45ffd3b3b33", "packages": [ { "name": "adhocore/jwt", @@ -4810,16 +4810,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "dev-fix-attribute-creation-errors", + "version": "0.41.9", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "a94668e92f9d8dd23770e1075076b2b9534f2886" + "reference": "61037c1ed9262308cab49c1d760f3278036ab694" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/a94668e92f9d8dd23770e1075076b2b9534f2886", - "reference": "a94668e92f9d8dd23770e1075076b2b9534f2886", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/61037c1ed9262308cab49c1d760f3278036ab694", + "reference": "61037c1ed9262308cab49c1d760f3278036ab694", "shasum": "" }, "require": { @@ -4855,9 +4855,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/fix-attribute-creation-errors" + "source": "https://github.com/appwrite/sdk-generator/tree/0.41.9" }, - "time": "2025-06-27T06:11:13+00:00" + "time": "2025-06-27T10:16:17+00:00" }, { "name": "doctrine/annotations", @@ -8236,9 +8236,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "appwrite/sdk-generator": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index ecb46de057..f5ac47a832 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -323,7 +323,8 @@ services: depends_on: - redis - mariadb - - request-catcher + - request-catcher-sms + - request-catcher-webhook environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1075,15 +1076,24 @@ services: networks: - appwrite - request-catcher: # used mainly for dev tests + request-catcher-webhook: # used mainly for dev tests (mock HTTP webhook) image: appwrite/requestcatcher:1.0.0 - container_name: appwrite-requestcatcher + container_name: appwrite-requestcatcher-webhook <<: *x-logging ports: - "9504:5000" networks: - appwrite + request-catcher-sms: # used mainly for dev tests (mock SMS auth secret) + image: appwrite/requestcatcher:1.0.0 + container_name: appwrite-requestcatcher-sms + <<: *x-logging + ports: + - "9507:5000" + networks: + - appwrite + adminer: image: adminer container_name: appwrite-adminer diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 71b932590d..e58f2b8664 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -235,15 +235,15 @@ class Base extends Action fn () => $dbForPlatform->createDocument('rules', new Document([ '$id' => $ruleId, 'projectId' => $project->getId(), - 'projectInternalId' => $project->getInternalId(), + 'projectInternalId' => $project->getSequence(), 'domain' => $domain, 'type' => 'deployment', 'trigger' => 'deployment', 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentInternalId' => $deployment->getSequence(), 'deploymentResourceType' => 'site', 'deploymentResourceId' => $site->getId(), - 'deploymentResourceInternalId' => $site->getInternalId(), + 'deploymentResourceInternalId' => $site->getSequence(), 'deploymentVcsProviderBranch' => $providerBranch, 'status' => 'verified', 'certificateId' => '', @@ -272,15 +272,15 @@ class Base extends Action fn () => $dbForPlatform->createDocument('rules', new Document([ '$id' => $ruleId, 'projectId' => $project->getId(), - 'projectInternalId' => $project->getInternalId(), + 'projectInternalId' => $project->getSequence(), 'domain' => $domain, 'type' => 'deployment', 'trigger' => 'deployment', 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentInternalId' => $deployment->getSequence(), 'deploymentResourceType' => 'site', 'deploymentResourceId' => $site->getId(), - 'deploymentResourceInternalId' => $site->getInternalId(), + 'deploymentResourceInternalId' => $site->getSequence(), 'deploymentVcsProviderBranch' => $providerBranch, 'status' => 'verified', 'certificateId' => '', diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index ac83248049..0369879a94 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -747,6 +747,13 @@ class Builds extends Action if ($separator !== false) { $logs = \substr($logs, 0, $separator); $insideSeparation = true; + + $leftover = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_START}')); + $separator = \strpos($leftover, '{APPWRITE_DETECTION_SEPARATOR_END}'); + if ($separator !== false) { + $logs .= \substr($leftover, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); + $insideSeparation = false; + } } } else { $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_END}'); @@ -829,18 +836,10 @@ class Builds extends Action // Separate logs for SSR detection $detectionLogs = ''; - $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); - if ($separator !== false) { - $detectionLogs = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR}')); - $separatorEnd = \strpos($detectionLogs, '{APPWRITE_DETECTION_SEPARATOR_END}'); - $logs .= \substr($detectionLogs, $separatorEnd + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); - $detectionLogs = \substr($detectionLogs, 0, $separatorEnd); - $logs = \substr($logs, 0, $separator); - } - - if ($resource->getCollection() === 'sites') { - $date = \date('H:i:s'); - $logs .= "[$date] [appwrite] Screenshot capturing started. \n"; + if (\str_contains($logs, '{APPWRITE_DETECTION_SEPARATOR_START}')) { + [$logsBefore, $detectionLogsStart] = \explode('{APPWRITE_DETECTION_SEPARATOR_START}', $logs, 2); + [$detectionLogs, $logsAfter] = \explode('{APPWRITE_DETECTION_SEPARATOR_END}', $detectionLogsStart, 2); + $logs = ($logsBefore ?? '') . ($logsAfter ?? ''); } $deployment->setAttribute('buildLogs', $logs); @@ -870,16 +869,24 @@ class Builds extends Action } } - $deployment->setAttribute('buildLogs', $logs); - - $this->afterBuildSuccess($dbForProject, $deployment); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - $queueForRealtime ->setPayload($deployment->getArrayCopy()) ->trigger(); + $this->afterBuildSuccess($queueForRealtime, $dbForProject, $deployment); + $logs = $deployment->getAttribute('buildLogs', ''); + + if ($resource->getCollection() === 'sites') { + $date = \date('H:i:s'); + $logs .= "[$date] [appwrite] Screenshot capturing started. \n"; + $deployment->setAttribute('buildLogs', $logs); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + $queueForRealtime + ->setPayload($deployment->getArrayCopy()) + ->trigger(); + } + /** Screenshot site */ if ($resource->getCollection() === 'sites') { try { @@ -1227,12 +1234,8 @@ class Builds extends Action $message = "" . $message; } - $separator = \strpos($message, '{APPWRITE_DETECTION_SEPARATOR_START}'); - if ($separator !== false) { - $error = \substr($message, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_START}')); - $message = \substr($message, 0, $separator); - $message .= "\n" . $error; - } + $message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_START}', '', $message); + $message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_END}', '', $message); // Combine with previous logs if deployment got past build process $previousLogs = ''; @@ -1321,12 +1324,14 @@ class Builds extends Action /** * Hook to run after build success * + * @param Realtime $queueForRealtime * @param Database $dbForProject * @param Document $deployment * @return void */ - protected function afterBuildSuccess(Database $dbForProject, Document &$deployment): void + protected function afterBuildSuccess(Realtime $queueForRealtime, Database $dbForProject, Document &$deployment): void { + assert($queueForRealtime instanceof Realtime); assert($dbForProject instanceof Database); assert($deployment instanceof Document); } diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 35733200c7..668993fdae 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -426,7 +426,7 @@ class Messaging extends Action $credentials = $provider->getAttribute('credentials'); return match ($provider->getAttribute('provider')) { - 'mock' => new Mock('username', 'password'), + 'mock' => (new Mock('username', 'password'))->setEndpoint('http://request-catcher-sms:5000/'), 'twilio' => new Twilio( $credentials['accountSid'] ?? '', $credentials['authToken'] ?? '', diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index e0e40f0169..df4c5fd8f5 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -188,9 +188,9 @@ class Executor array $headers, float $cpus, int $memory, - string $runtimeEntrypoint = '', bool $logging, - int $requestTimeout = null + string $runtimeEntrypoint = '', + ?int $requestTimeout = null ) { if (empty($headers['host'])) { $headers['host'] = System::getEnv('_APP_DOMAIN', ''); diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 620409bb39..bce6e7206f 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -60,7 +60,7 @@ class HTTPTest extends Scope 'origin' => 'http://localhost', ])); - $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(200, $response['headers']['status-code'], "Simple GET /robots.txt HTTP request failed: " . \json_encode($response)); $this->assertStringContainsString('# robotstxt.org/', $response['body']); } diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index a354696f53..c5152adbcc 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -140,7 +140,7 @@ trait ProjectCustom 'teams.*', 'users.*' ], - 'url' => 'http://request-catcher:5000/webhook', + 'url' => 'http://request-catcher-webhook:5000/', 'security' => false, ]); diff --git a/tests/e2e/Scopes/Scope.php b/tests/e2e/Scopes/Scope.php index dddc97e3a0..2ee0d0198e 100644 --- a/tests/e2e/Scopes/Scope.php +++ b/tests/e2e/Scopes/Scope.php @@ -13,6 +13,9 @@ abstract class Scope extends TestCase use Retryable; use Async; + public const REQUEST_TYPE_WEBHOOK = 'webhook'; + public const REQUEST_TYPE_SMS = 'sms'; + protected ?Client $client = null; protected string $endpoint = 'http://localhost/v1'; @@ -76,10 +79,16 @@ abstract class Scope extends TestCase return []; } - protected function assertLastRequest(callable $probe, $timeoutMs = 20_000, $waitMs = 500): array + protected function assertLastRequest(callable $probe, string $type, $timeoutMs = 20_000, $waitMs = 500): array { - $this->assertEventually(function () use (&$request, $probe) { - $request = json_decode(file_get_contents('http://request-catcher:5000/__last_request__'), true); + $hostname = match ($type) { + 'webhook' => 'request-catcher-webhook', + 'sms' => 'request-catcher-sms', + default => throw new \Exception('Invalid request catcher type.'), + }; + + $this->assertEventually(function () use (&$request, $probe, $hostname) { + $request = json_decode(file_get_contents('http://' . $hostname . ':5000/__last_request__'), true); $request['data'] = json_decode($request['data'], true); call_user_func($probe, $request); @@ -88,11 +97,16 @@ abstract class Scope extends TestCase return $request; } + /** + * @deprecated Use assertLastRequest instead. Used only historically in webhook tests + */ protected function getLastRequest(): array { + $hostname = 'request-catcher-webhook'; + sleep(2); - $request = json_decode(file_get_contents('http://request-catcher:5000/__last_request__'), true); + $request = json_decode(file_get_contents('http://' . $hostname . ':5000/__last_request__'), true); $request['data'] = json_decode($request['data'], true); return $request; diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 86c78d5d81..1b84ce7083 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2065,14 +2065,13 @@ class AccountCustomClientTest extends Scope $userId = $response['body']['userId']; $smsRequest = $this->assertLastRequest(function (array $request) use ($number) { - $this->assertEquals('http://request-catcher:5000/mock-sms', $request['url']); $this->assertEquals('Appwrite Mock Message Sender', $request['headers']['User-Agent']); $this->assertEquals('username', $request['headers']['X-Username']); $this->assertEquals('password', $request['headers']['X-Key']); $this->assertEquals('POST', $request['method']); $this->assertEquals('+123456789', $request['data']['from']); $this->assertEquals($number, $request['data']['to']); - }); + }, Scope::REQUEST_TYPE_SMS); $data['token'] = $smsRequest['data']['message']; $data['id'] = $userId; @@ -2416,13 +2415,21 @@ class AccountCustomClientTest extends Scope $this->assertEquals(201, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); + $this->assertNotEmpty($response['body']['$createdAt']); $this->assertEmpty($response['body']['secret']); $this->assertTrue((new DatetimeValidator())->isValid($response['body']['expire'])); - $smsRequest = $this->assertLastRequest(function ($request) { + $tokenCreatedAt = $response['body']['$createdAt']; + + $smsRequest = $this->assertLastRequest(function ($request) use ($tokenCreatedAt) { $this->assertArrayHasKey('data', $request); + $this->assertArrayHasKey('time', $request); $this->assertArrayHasKey('message', $request['data'], "Last request missing message: " . \json_encode($request)); - }); + + // Ensure we are not using token from last sms login + $tokenRecievedAt = $request['time']; + $this->assertGreaterThan($tokenCreatedAt, $tokenRecievedAt); + }, Scope::REQUEST_TYPE_SMS); /** * Test for FAILURE