From 27a5bd70bc5d4bfb1224c480a455ac1242e8bde2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Sun, 18 May 2025 00:56:11 +0100 Subject: [PATCH 1/6] fix: remove builds --- src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index da6f006265..c7fc833316 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -671,7 +671,7 @@ class Builds extends Action cpus: $cpus, memory: $memory, timeout: $timeout, - remove: false, + remove: true, entrypoint: $deployment->getAttribute('entrypoint', ''), destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, From 03f01e322e9239134d188c7f946f74a52e7dcf22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 May 2025 10:30:27 +0200 Subject: [PATCH 2/6] SSR detection without createCommand and remove=false --- .../Modules/Functions/Workers/Builds.php | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index c7fc833316..1b3e1a8af9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -659,6 +659,11 @@ class Builds extends Action if ($version === 'v2') { $command = 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh'; } else { + if ($resource->getCollection() === 'sites') { + $listFilesCommand = 'echo "{APPWRITE_DETECTION_SEPARATOR}" && cd /usr/local/build && cd $OPEN_RUNTIMES_OUTPUT_DIRECTORY && find . -name \'node_modules\' -prune -o -type f -print'; + $command .= '&& ' . $listFilesCommand; + } + $command = 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh ' . \trim(\escapeshellarg($command)); } @@ -706,6 +711,12 @@ class Builds extends Action // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors $logs = \mb_substr($logs, 0, null, 'UTF-8'); + // Do not stream logs added for SSR detection + $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR}'); + if ($separator !== false) { + $logs = substr($logs, 0, $separator); + } + $currentLogs = $deployment->getAttribute('buildLogs', ''); $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); @@ -755,44 +766,6 @@ class Builds extends Action throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.'); } - if ($resource->getCollection() === 'sites') { - // TODO: Refactor with structured command in future, using utopia library (CLI) - $listFilesCommand = "cd /usr/local/build && cd " . \escapeshellarg($resource->getAttribute('outputDirectory', './')) . " && find . -name 'node_modules' -prune -o -type f -print"; - $command = $executor->createCommand( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - command: $listFilesCommand, - timeout: 15 - ); - - $files = \explode("\n", $command['output']); // Parse output - $files = \array_filter($files); // Remove empty - $files = \array_map(fn ($file) => \trim($file), $files); // Remove whitepsaces - $files = \array_map(fn ($file) => \str_starts_with($file, './') ? \substr($file, 2) : $file, $files); // Remove beginning ./ - - $detector = new Rendering($files, $resource->getAttribute('framework', '')); - $detector - ->addOption(new SSR()) - ->addOption(new XStatic()); - $detection = $detector->detect(); - - $adapter = $resource->getAttribute('adapter', ''); - - if (empty($adapter)) { - $resource->setAttribute('adapter', $detection->getName()); - $resource->setAttribute('fallbackFile', $detection->getFallbackFile() ?? ''); - $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); - - $deployment->setAttribute('adapter', $detection->getName()); - $deployment->setAttribute('fallbackFile', $detection->getFallbackFile() ?? ''); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } elseif ($adapter === 'ssr' && $detection->getName() === 'static') { - throw new \Exception('Adapter mismatch. Detected: ' . $detection->getName() . ' does not match with the set adapter: ' . $adapter); - } - } - - $executor->deleteRuntime($project->getId(), $deployment->getId(), '-build'); - /** Update the build document */ $deployment->setAttribute('buildStartedAt', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); $deployment->setAttribute('buildEndedAt', $endTime); @@ -806,6 +779,14 @@ class Builds extends Action $logs .= $log['content']; } + // Separate logs for SSR detection + $detectionLogs = ''; + $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR}'); + if ($separator !== false) { + $detectionLogs = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR}')); + $logs = \substr($logs, 0, $separator); + } + if ($resource->getCollection() === 'sites') { $date = \date('H:i:s'); $logs .= "[$date] [appwrite] Screenshot capturing started. \n"; @@ -813,6 +794,31 @@ class Builds extends Action $deployment->setAttribute('buildLogs', $logs); + if ($resource->getCollection() === 'sites' && !empty($detectionLogs)) { + $files = \explode("\n", $detectionLogs); // Parse output + $files = \array_filter($files); // Remove empty + $files = \array_map(fn ($file) => \trim($file), $files); // Remove whitepsaces + $files = \array_map(fn ($file) => \str_starts_with($file, './') ? \substr($file, 2) : $file, $files); // Remove beginning ./ + + $detector = new Rendering($files, $resource->getAttribute('framework', '')); + $detector + ->addOption(new SSR()) + ->addOption(new XStatic()); + $detection = $detector->detect(); + + $adapter = $resource->getAttribute('adapter', ''); + if (empty($adapter)) { + $resource->setAttribute('adapter', $detection->getName()); + $resource->setAttribute('fallbackFile', $detection->getFallbackFile() ?? ''); + $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); + + $deployment->setAttribute('adapter', $detection->getName()); + $deployment->setAttribute('fallbackFile', $detection->getFallbackFile() ?? ''); + } elseif ($adapter === 'ssr' && $detection->getName() === 'static') { + throw new \Exception('Adapter mismatch. Detected: ' . $detection->getName() . ' does not match with the set adapter: ' . $adapter); + } + } + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); $queueForRealtime From 9b4f787d404075da943608d3f9ee3ee1a356be40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 May 2025 11:08:14 +0200 Subject: [PATCH 3/6] Fix edge case with no commands --- src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 1b3e1a8af9..dff7e7f3f3 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -661,7 +661,12 @@ class Builds extends Action } else { if ($resource->getCollection() === 'sites') { $listFilesCommand = 'echo "{APPWRITE_DETECTION_SEPARATOR}" && cd /usr/local/build && cd $OPEN_RUNTIMES_OUTPUT_DIRECTORY && find . -name \'node_modules\' -prune -o -type f -print'; - $command .= '&& ' . $listFilesCommand; + + if (empty($command)) { + $command = $listFilesCommand; + } else { + $command .= ' && ' . $listFilesCommand; + } } $command = 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh ' . \trim(\escapeshellarg($command)); From 0e699290547c810fa16c2db41d9baf3be63e4fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 May 2025 11:39:30 +0200 Subject: [PATCH 4/6] Fix separation strategy --- .../Modules/Functions/Workers/Builds.php | 39 ++++++++++++++++--- .../Services/Sites/SitesCustomServerTest.php | 1 + 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index dff7e7f3f3..a1bf59fb02 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -660,8 +660,20 @@ class Builds extends Action $command = 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh'; } else { if ($resource->getCollection() === 'sites') { - $listFilesCommand = 'echo "{APPWRITE_DETECTION_SEPARATOR}" && cd /usr/local/build && cd $OPEN_RUNTIMES_OUTPUT_DIRECTORY && find . -name \'node_modules\' -prune -o -type f -print'; + $listFilesCommand = ''; + // Start separation, enter build folder + $listFilesCommand .= 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; + + // Enter output directory, if set + if (!empty($resource->getAttribute('outputDirectory', ''))) { + $listFilesCommand .= 'cd ' . \escapeshellarg($resource->getAttribute('outputDirectory', '')); + } + + // Print files, and end separation + $listFilesCommand .= 'find . -name \'node_modules\' -prune -o -type f -print && echo "{APPWRITE_DETECTION_SEPARATOR_END}"'; + + // Use SSR file listing if (empty($command)) { $command = $listFilesCommand; } else { @@ -694,11 +706,13 @@ class Builds extends Action }), Co\go(function () use ($executor, $project, &$deployment, &$response, $dbForProject, $timeout, &$err, $queueForRealtime, &$isCanceled) { try { + $insideSeparation = false; + $executor->getLogs( deploymentId: $deployment->getId(), projectId: $project->getId(), timeout: $timeout, - callback: function ($logs) use (&$response, &$err, $dbForProject, &$isCanceled, &$deployment, $queueForRealtime) { + callback: function ($logs) use (&$response, &$err, $dbForProject, &$isCanceled, &$deployment, $queueForRealtime, &$insideSeparation) { if ($isCanceled) { return; } @@ -717,9 +731,19 @@ class Builds extends Action $logs = \mb_substr($logs, 0, null, 'UTF-8'); // Do not stream logs added for SSR detection - $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR}'); - if ($separator !== false) { - $logs = substr($logs, 0, $separator); + if (!$insideSeparation) { + $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); + if ($separator !== false) { + $logs = \substr($logs, 0, $separator); + $insideSeparation = true; + } + } else { + $logs = ''; + $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_END}'); + if ($separator !== false) { + $logs = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); + $insideSeparation = false; + } } $currentLogs = $deployment->getAttribute('buildLogs', ''); @@ -786,9 +810,12 @@ class Builds extends Action // Separate logs for SSR detection $detectionLogs = ''; - $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR}'); + $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); if ($separator !== false) { $detectionLogs = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR}')); + $separatorEnd = \strpos($detectionLogs, '{APPWRITE_DETECTION_SEPARATOR_END}'); + $logs .= \substr($detectionLogs, $separatorEnd + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); + $detectionLogs = \substr($detectionLogs, 0, $separatorEnd); $logs = \substr($logs, 0, $separator); } diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index f9d5b4acdf..7a73e29b02 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -524,6 +524,7 @@ class SitesCustomServerTest extends Scope $this->assertNotEmpty($deploymentId); $site = $this->getSite($siteId); + \var_dump($site); $this->assertEquals('200', $site['headers']['status-code']); $this->assertEquals('static', $site['body']['adapter']); $this->assertEquals('main.html', $site['body']['fallbackFile']); From b228bdd9c7fcad2b64d63ac8e3239e574b9ba2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 May 2025 12:02:32 +0200 Subject: [PATCH 5/6] Fix failing tests --- src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 4 ++-- tests/e2e/Services/Sites/SitesCustomServerTest.php | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index a1bf59fb02..0bb84c6b4e 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -667,11 +667,11 @@ class Builds extends Action // Enter output directory, if set if (!empty($resource->getAttribute('outputDirectory', ''))) { - $listFilesCommand .= 'cd ' . \escapeshellarg($resource->getAttribute('outputDirectory', '')); + $listFilesCommand .= ' && cd ' . \escapeshellarg($resource->getAttribute('outputDirectory', '')); } // Print files, and end separation - $listFilesCommand .= 'find . -name \'node_modules\' -prune -o -type f -print && echo "{APPWRITE_DETECTION_SEPARATOR_END}"'; + $listFilesCommand .= ' && find . -name \'node_modules\' -prune -o -type f -print && echo "{APPWRITE_DETECTION_SEPARATOR_END}"'; // Use SSR file listing if (empty($command)) { diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 7a73e29b02..f9d5b4acdf 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -524,7 +524,6 @@ class SitesCustomServerTest extends Scope $this->assertNotEmpty($deploymentId); $site = $this->getSite($siteId); - \var_dump($site); $this->assertEquals('200', $site['headers']['status-code']); $this->assertEquals('static', $site['body']['adapter']); $this->assertEquals('main.html', $site['body']['fallbackFile']); From 2bee98afcf8e8bb404c364073e77d3051311d0c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 May 2025 12:51:07 +0200 Subject: [PATCH 6/6] Fix remaining tests --- .../Modules/Functions/Workers/Builds.php | 30 +++++++++++++++---- .../Realtime/RealtimeConsoleClientTest.php | 18 +++++------ .../Services/Sites/SitesCustomServerTest.php | 2 +- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 0bb84c6b4e..27c6e59128 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -693,7 +693,7 @@ class Builds extends Action cpus: $cpus, memory: $memory, timeout: $timeout, - remove: true, + remove: true, entrypoint: $deployment->getAttribute('entrypoint', ''), destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, @@ -746,7 +746,12 @@ class Builds extends Action } } + if (empty($logs)) { + return; + } + $currentLogs = $deployment->getAttribute('buildLogs', ''); + $affected = false; $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); foreach (\explode("\n", $streamLogs) as $streamLog) { @@ -759,14 +764,20 @@ class Builds extends Action // TODO: use part[0] as timestamp when switching to dbForLogs for build logs $currentLogs .= $streamParts[1]; + + if (!empty($streamParts[1])) { + $affected = true; + } } - $deployment = $deployment->setAttribute('buildLogs', $currentLogs); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + if ($affected) { + $deployment = $deployment->setAttribute('buildLogs', $currentLogs); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); + $queueForRealtime + ->setPayload($deployment->getArrayCopy()) + ->trigger(); + } } } ); @@ -1199,6 +1210,13 @@ class Builds extends Action $message = "" . $message; } + $separator = \strpos($message, '{APPWRITE_DETECTION_SEPARATOR_START}'); + if ($separator !== false) { + $error = \substr($message, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_START}')); + $message = \substr($message, 0, $separator); + $message .= "\n" . $error; + } + $endTime = DateTime::now(); $durationEnd = \microtime(true); $deployment->setAttribute('buildEndedAt', $endTime); diff --git a/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php b/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php index 01de2782a5..ba2d18694a 100644 --- a/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php @@ -607,15 +607,6 @@ class RealtimeConsoleClientTest extends Scope $this->assertContains("projects.{$projectId}", $response['data']['channels']); $this->assertArrayHasKey('buildLogs', $response['data']['payload']); - // Ignore comparasion for first payload - if ($previousBuildLogs !== null) { - $this->assertNotEquals($previousBuildLogs, $response['data']['payload']['buildLogs']); - } - - $previousBuildLogs = $response['data']['payload']['buildLogs']; - - $this->assertEquals('building', $response['data']['payload']['status']); - if (!empty($response['data']['payload']['buildEndedAt'])) { $this->assertNotEmpty($response['data']['payload']['buildEndedAt']); $this->assertNotEmpty($response['data']['payload']['buildStartedAt']); @@ -626,6 +617,15 @@ class RealtimeConsoleClientTest extends Scope $this->assertNotEmpty($response['data']['payload']['buildLogs']); break; } + + // Ignore comparasion for first payload + if ($previousBuildLogs !== null) { + $this->assertNotEquals($previousBuildLogs, $response['data']['payload']['buildLogs']); + } + + $previousBuildLogs = $response['data']['payload']['buildLogs']; + + $this->assertEquals('building', $response['data']['payload']['status']); } $response = json_decode($client->receive(), true); diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index f9d5b4acdf..dd4efa5932 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -2646,7 +2646,7 @@ class SitesCustomServerTest extends Scope $this->assertEventually(function () use ($siteId, $deploymentId) { $deployment = $this->getDeployment($siteId, $deploymentId); $this->assertEquals('failed', $deployment['body']['status'], 'Deployment status does not match: ' . json_encode($deployment['body'], JSON_PRETTY_PRINT)); - $this->assertStringContainsString('Error:', $deployment['body']['buildLogs'], 'Deployment logs do not match: ' . json_encode($deployment['body'], JSON_PRETTY_PRINT)); + $this->assertStringContainsString('No such file or directory', $deployment['body']['buildLogs'], 'Deployment logs do not match: ' . json_encode($deployment['body'], JSON_PRETTY_PRINT)); }, 100000, 500); $this->cleanupSite($siteId);