From ab03b4e3caa62347066912d520073eefec6abee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 26 Mar 2025 12:59:05 +0100 Subject: [PATCH 1/9] Deployment ready after screenshot --- app/config/frameworks.php | 2 +- app/controllers/general.php | 5 +- src/Appwrite/Auth/Key.php | 10 +++- .../Modules/Functions/Workers/Builds.php | 48 +++++++++++++------ 4 files changed, 46 insertions(+), 19 deletions(-) diff --git a/app/config/frameworks.php b/app/config/frameworks.php index 82fd70bf5b..5440e9a538 100644 --- a/app/config/frameworks.php +++ b/app/config/frameworks.php @@ -138,7 +138,7 @@ return [ 'vue' => [ 'key' => 'vue', 'name' => 'Vue.js', - 'screenshotSleep' => 3000, + 'screenshotSleep' => 5000, 'buildRuntime' => 'node-22', 'runtimes' => getVersions($templateRuntimes['NODE']['versions'], 'node'), 'adapters' => [ diff --git a/app/controllers/general.php b/app/controllers/general.php index 3f105484e1..13dec5dac9 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -268,8 +268,9 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw if (\is_null($runtime)) { throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); } - - if ($deployment->getAttribute('status') !== 'ready') { + + $allowAnyStatus = !\is_null($apiKey) && $apiKey->isDeploymentStatusIgnored(); + if (!$allowAnyStatus && $deployment->getAttribute('status') !== 'ready') { throw new AppwriteException(AppwriteException::BUILD_NOT_READY); } diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 89c28c4727..87e77b2e06 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -24,6 +24,7 @@ class Key protected bool $bannerDisabled = false, protected bool $projectCheckDisabled = false, protected bool $previewAuthDisabled = false, + protected bool $deploymentStatusIgnored = false, ) { } @@ -78,6 +79,11 @@ class Key { return $this->previewAuthDisabled; } + + public function isDeploymentStatusIgnored(): bool + { + return $this->deploymentStatusIgnored; + } public function isProjectCheckDisabled(): bool { @@ -139,6 +145,7 @@ class Key $bannerDisabled = $payload['bannerDisabled'] ?? false; $projectCheckDisabled = $payload['projectCheckDisabled'] ?? false; $previewAuthDisabled = $payload['previewAuthDisabled'] ?? false; + $deploymentStatusIgnored = $payload['deploymentStatusIgnored'] ?? false; $scopes = \array_merge($payload['scopes'] ?? [], $scopes); if (!$projectCheckDisabled && $projectId !== $project->getId()) { @@ -156,7 +163,8 @@ class Key $hostnameOverride, $bannerDisabled, $projectCheckDisabled, - $previewAuthDisabled + $previewAuthDisabled, + $deploymentStatusIgnored ); case API_KEY_STANDARD: $key = $project->find( diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 3f28e7a7ce..849fd514d0 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -778,24 +778,26 @@ class Builds extends Action } $executor->deleteRuntime($project->getId(), $deployment->getId(), '-build'); - + /** Update the build document */ $deployment->setAttribute('buildStartAt', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); $deployment->setAttribute('buildEndAt', $endTime); $deployment->setAttribute('buildDuration', \intval(\ceil($durationEnd - $durationStart))); - $deployment->setAttribute('status', 'ready'); $deployment->setAttribute('buildPath', $response['path']); $deployment->setAttribute('buildSize', $response['size']); $deployment->setAttribute('totalSize', $deployment->getAttribute('buildSize', 0) + $deployment->getAttribute('sourceSize', 0)); - + $logs = ''; foreach ($response['output'] as $log) { $logs .= $log['content']; } + $logs .= "Capturing screenshots ...\n"; $deployment->setAttribute('buildLogs', $logs); + + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); - + if ($deployment->getInternalId() === $resource->getAttribute('latestDeploymentInternalId', '')) { $resource = $resource->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), $resource); @@ -804,13 +806,8 @@ class Builds extends Action $queueForRealtime ->setPayload($deployment->getArrayCopy()) ->trigger(); - - if ($isVcsEnabled) { - $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); - } - - Console::success("Build id: $deploymentId created"); - + + /** Screenshot site */ if ($resource->getCollection() === 'sites') { try { $rule = Authorization::skip(fn () => $dbForPlatform->findOne('rules', [ @@ -848,6 +845,7 @@ class Builds extends Action 'bannerDisabled' => true, 'projectCheckDisabled' => true, 'previewAuthDisabled' => true, + 'deploymentStatusIgnored' => true ]); // TODO: @Meldiron if becomes too slow, do concurrently @@ -864,17 +862,17 @@ class Builds extends Action $config['sleep'] = $framework['screenshotSleep']; } - $response = $client->fetch( + $fetchResponse = $client->fetch( url: 'http://appwrite-browser:3000/v1/screenshots', method: 'POST', body: $config ); - if ($response->getStatusCode() >= 400) { - throw new \Exception($response->getBody()); + if ($fetchResponse->getStatusCode() >= 400) { + throw new \Exception($fetchResponse->getBody()); } - $screenshot = $response->getBody(); + $screenshot = $fetchResponse->getBody(); $fileId = ID::unique(); $fileName = $fileId . '.png'; @@ -928,6 +926,20 @@ class Builds extends Action } } + /** Update the status */ + $deployment->setAttribute('status', 'ready'); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); + + $queueForRealtime + ->setPayload($deployment->getArrayCopy()) + ->trigger(); + + if ($isVcsEnabled) { + $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); + } + + Console::success("Build id: $deploymentId created"); + /** Set auto deploy */ if ($deployment->getAttribute('activate') === true) { $resource->setAttribute('live', true); @@ -1066,6 +1078,12 @@ class Builds extends Action Authorization::skip(fn () => $dbForPlatform->updateDocument('schedules', $schedule->getId(), $schedule)); } } catch (\Throwable $th) { + Console::warning('Build failed:'); + Console::error($th->getMessage()); + Console::error($th->getFile()); + Console::error($th->getLine()); + Console::error($th->getTraceAsString()); + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; From 681b6031aa5dfc85bd78796d7f05e85a339e7023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 26 Mar 2025 12:59:15 +0100 Subject: [PATCH 2/9] Linter fix --- app/controllers/general.php | 2 +- src/Appwrite/Auth/Key.php | 2 +- .../Platform/Modules/Functions/Workers/Builds.php | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 13dec5dac9..006584c954 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -268,7 +268,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw if (\is_null($runtime)) { throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); } - + $allowAnyStatus = !\is_null($apiKey) && $apiKey->isDeploymentStatusIgnored(); if (!$allowAnyStatus && $deployment->getAttribute('status') !== 'ready') { throw new AppwriteException(AppwriteException::BUILD_NOT_READY); diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 87e77b2e06..44a75a6ee3 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -79,7 +79,7 @@ class Key { return $this->previewAuthDisabled; } - + public function isDeploymentStatusIgnored(): bool { return $this->deploymentStatusIgnored; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 849fd514d0..b18b6e4004 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -778,7 +778,7 @@ class Builds extends Action } $executor->deleteRuntime($project->getId(), $deployment->getId(), '-build'); - + /** Update the build document */ $deployment->setAttribute('buildStartAt', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); $deployment->setAttribute('buildEndAt', $endTime); @@ -786,18 +786,18 @@ class Builds extends Action $deployment->setAttribute('buildPath', $response['path']); $deployment->setAttribute('buildSize', $response['size']); $deployment->setAttribute('totalSize', $deployment->getAttribute('buildSize', 0) + $deployment->getAttribute('sourceSize', 0)); - + $logs = ''; foreach ($response['output'] as $log) { $logs .= $log['content']; } $logs .= "Capturing screenshots ...\n"; $deployment->setAttribute('buildLogs', $logs); - + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); - + if ($deployment->getInternalId() === $resource->getAttribute('latestDeploymentInternalId', '')) { $resource = $resource->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), $resource); @@ -806,7 +806,7 @@ class Builds extends Action $queueForRealtime ->setPayload($deployment->getArrayCopy()) ->trigger(); - + /** Screenshot site */ if ($resource->getCollection() === 'sites') { try { @@ -1083,7 +1083,7 @@ class Builds extends Action Console::error($th->getFile()); Console::error($th->getLine()); Console::error($th->getTraceAsString()); - + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; From 2c55f5aaa607f01e12fe0680a539100f5f18f6ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 26 Mar 2025 13:20:14 +0100 Subject: [PATCH 3/9] Concurrency screenshots --- .../Modules/Functions/Workers/Builds.php | 117 ++++++++++-------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index b18b6e4004..eecc439642 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -41,6 +41,8 @@ use Utopia\Storage\Device\Local; use Utopia\System\System; use Utopia\VCS\Adapter\Git\GitHub; +use function Swoole\Coroutine\batch; + class Builds extends Action { public static function getName(): string @@ -849,69 +851,76 @@ class Builds extends Action ]); // TODO: @Meldiron if becomes too slow, do concurrently - foreach ($configs as $key => $config) { - $config['headers'] = \array_merge($config['headers'] ?? [], [ - 'x-appwrite-key' => API_KEY_DYNAMIC . '_' . $apiKey - ]); + $screenshots = batch(\array_map(function ($key) use ($configs, $deviceForFiles, $apiKey, $resource, $client, $bucket, $project, $dbForPlatform) { + return function () use ($key, $configs, $deviceForFiles, $apiKey, $resource, $client, $bucket, $project, $dbForPlatform) { + $config = $configs[$key]; - $config['sleep'] = 3000; + $config['headers'] = \array_merge($config['headers'] ?? [], [ + 'x-appwrite-key' => API_KEY_DYNAMIC . '_' . $apiKey + ]); + $config['sleep'] = 3000; - $frameworks = Config::getParam('frameworks', []); - $framework = $frameworks[$resource->getAttribute('framework', '')] ?? null; - if (!is_null($framework)) { - $config['sleep'] = $framework['screenshotSleep']; - } + $frameworks = Config::getParam('frameworks', []); + $framework = $frameworks[$resource->getAttribute('framework', '')] ?? null; + if (!is_null($framework)) { + $config['sleep'] = $framework['screenshotSleep']; + } - $fetchResponse = $client->fetch( - url: 'http://appwrite-browser:3000/v1/screenshots', - method: 'POST', - body: $config - ); + $fetchResponse = $client->fetch( + url: 'http://appwrite-browser:3000/v1/screenshots', + method: 'POST', + body: $config + ); - if ($fetchResponse->getStatusCode() >= 400) { - throw new \Exception($fetchResponse->getBody()); - } + if ($fetchResponse->getStatusCode() >= 400) { + throw new \Exception($fetchResponse->getBody()); + } - $screenshot = $fetchResponse->getBody(); + $screenshot = $fetchResponse->getBody(); - $fileId = ID::unique(); - $fileName = $fileId . '.png'; - $path = $deviceForFiles->getPath($fileName); - $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root - $success = $deviceForFiles->write($path, $screenshot, "image/png"); + $fileId = ID::unique(); + $fileName = $fileId . '.png'; + $path = $deviceForFiles->getPath($fileName); + $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + $success = $deviceForFiles->write($path, $screenshot, "image/png"); - if (!$success) { - throw new \Exception("Screenshot failed to save"); - } + if (!$success) { + throw new \Exception("Screenshot failed to save"); + } - $teamId = $project->getAttribute('teamId', ''); - $file = new Document([ - '$id' => $fileId, - '$permissions' => [ - Permission::read(Role::team(ID::custom($teamId))), - ], - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getInternalId(), - 'name' => $fileName, - 'path' => $path, - 'signature' => $deviceForFiles->getFileHash($path), - 'mimeType' => $deviceForFiles->getFileMimeType($path), - 'sizeOriginal' => \strlen($screenshot), - 'sizeActual' => $deviceForFiles->getFileSize($path), - 'algorithm' => Compression::GZIP, - 'comment' => '', - 'chunksTotal' => 1, - 'chunksUploaded' => 1, - 'openSSLVersion' => null, - 'openSSLCipher' => null, - 'openSSLTag' => null, - 'openSSLIV' => null, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => ['content_type' => $deviceForFiles->getFileMimeType($path)], - ]); - $file = Authorization::skip(fn () => $dbForPlatform->createDocument('bucket_' . $bucket->getInternalId(), $file)); + $teamId = $project->getAttribute('teamId', ''); + $file = new Document([ + '$id' => $fileId, + '$permissions' => [ + Permission::read(Role::team(ID::custom($teamId))), + ], + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getInternalId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $deviceForFiles->getFileHash($path), + 'mimeType' => $deviceForFiles->getFileMimeType($path), + 'sizeOriginal' => \strlen($screenshot), + 'sizeActual' => $deviceForFiles->getFileSize($path), + 'algorithm' => Compression::GZIP, + 'comment' => '', + 'chunksTotal' => 1, + 'chunksUploaded' => 1, + 'openSSLVersion' => null, + 'openSSLCipher' => null, + 'openSSLTag' => null, + 'openSSLIV' => null, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => ['content_type' => $deviceForFiles->getFileMimeType($path)], + ]); + $file = Authorization::skip(fn () => $dbForPlatform->createDocument('bucket_' . $bucket->getInternalId(), $file)); - $deployment->setAttribute($key, $fileId); + return [ 'key' => $key, 'fileId' => $fileId ]; + }; + }, \array_keys($configs))); + + foreach ($screenshots as $screenshot) { + $deployment->setAttribute($screenshot['key'], $screenshot['fileId']); } $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); From 62be0e1d00eab357da3482080fb617496fb47728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 26 Mar 2025 14:38:18 +0100 Subject: [PATCH 4/9] Fix latest deployment attributes --- .../Platform/Modules/Functions/Workers/Builds.php | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index eecc439642..d2ef6deb36 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -798,13 +798,6 @@ class Builds extends Action $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); - - if ($deployment->getInternalId() === $resource->getAttribute('latestDeploymentInternalId', '')) { - $resource = $resource->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); - $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), $resource); - } - $queueForRealtime ->setPayload($deployment->getArrayCopy()) ->trigger(); @@ -938,6 +931,11 @@ class Builds extends Action /** Update the status */ $deployment->setAttribute('status', 'ready'); $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); + + if ($deployment->getInternalId() === $resource->getAttribute('latestDeploymentInternalId', '')) { + $resource = $resource->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); + $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), $resource); + } $queueForRealtime ->setPayload($deployment->getArrayCopy()) From 70fbf69c5689da448716d9f47c156cb0db60bd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 26 Mar 2025 14:38:27 +0100 Subject: [PATCH 5/9] Fix fonts --- app/views/general/404.phtml | 10 ++++++---- src/Appwrite/Transformation/Adapter/Preview.php | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/views/general/404.phtml b/app/views/general/404.phtml index 7ec1cfbf21..5e63344c8a 100644 --- a/app/views/general/404.phtml +++ b/app/views/general/404.phtml @@ -7,6 +7,8 @@ 404 Not Found