From 54d3668693860c1f3eacb908c38354f8ee65bc1a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:20:00 +0200 Subject: [PATCH 1/5] fix: todos for sites --- app/controllers/general.php | 144 +++++----- app/init.php | 10 +- .../Modules/Functions/Workers/Builds.php | 262 +++++++++++------- 3 files changed, 243 insertions(+), 173 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 88ece71e8a..adc2c82f49 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -54,19 +54,19 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $host = $request->getHostname() ?? ''; - $route = Authorization::skip( + $rule = Authorization::skip( fn () => $dbForConsole->find('rules', [ Query::equal('domain', [$host]), Query::limit(1) ]) )[0] ?? null; - if ($route === null) { - if ($host === System::getEnv('_APP_DOMAIN_FUNCTIONS', '')) { + if ($rule === null) { + if ($host === System::getEnv('_APP_DOMAIN_FUNCTIONS', '') || $host === System::getEnv('_APP_DOMAIN_SITES', '')) { throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'This domain cannot be used for security reasons. Please use any subdomain instead.'); } - if (\str_ends_with($host, System::getEnv('_APP_DOMAIN_FUNCTIONS', ''))) { + if (\str_ends_with($host, System::getEnv('_APP_DOMAIN_FUNCTIONS', '')) || \str_ends_with($host, System::getEnv('_APP_DOMAIN_SITES', ''))) { throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'This domain is not connected to any Appwrite resource yet. Please configure custom domain or function domain to allow this request.'); } @@ -78,13 +78,12 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo // Act as API - no Proxy logic $utopia->getRoute()?->label('error', ''); + return false; } - $projectId = $route->getAttribute('projectId'); - $project = Authorization::skip( - fn () => $dbForConsole->getDocument('projects', $projectId) - ); + $projectId = $rule->getAttribute('projectId'); + $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); if (array_key_exists('proxy', $project->getAttribute('services', []))) { $status = $project->getAttribute('services', [])['proxy']; if (!$status) { @@ -98,11 +97,15 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo return false; } - $type = $route->getAttribute('resourceType'); + $type = $rule->getAttribute('resourceType'); if ($type === 'function' || $type === 'site') { - $isFunction = $type === 'function' ; - $isSite = $type === 'site'; + // $isFunction = $type === 'function' ; + // $isSite = $type === 'site'; + $resourceCollection = match($type) { + 'function' => 'functions', + 'site' => 'sites' + }; $utopia->getRoute()?->label('sdk.namespace', 'functions'); $utopia->getRoute()?->label('sdk.method', 'createExecution'); @@ -116,8 +119,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } } - $resourceId = $route->getAttribute('resourceId'); - $projectId = $route->getAttribute('projectId'); + $resourceId = $rule->getAttribute('resourceId'); + $projectId = $rule->getAttribute('projectId'); $path = ($swooleRequest->server['request_uri'] ?? '/'); $query = ($swooleRequest->server['query_string'] ?? ''); @@ -135,21 +138,20 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); - $function = Authorization::skip(fn () => $dbForProject->getDocument($isSite ? 'sites' : 'functions', $resourceId)); + $resource = Authorization::skip(fn () => $dbForProject->getDocument($resourceCollection, $resourceId)); - if ($function->isEmpty() || !$function->getAttribute('enabled')) { + if ($resource->isEmpty() || !$resource->getAttribute('enabled')) { throw new AppwriteException(AppwriteException::FUNCTION_NOT_FOUND); } - $version = $function->getAttribute('version', 'v2'); + $version = $resource->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)]; + $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)]; - $runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null; - - // todo: fallback for static sites runtime - if ($isSite) { - $runtime = [ + //todo: have runtime configs for sites + $runtime = match($type) { + 'function' => (isset($runtimes[$resource->getAttribute('runtime', '')])) ? $runtimes[$resource->getAttribute('runtime', '')] : null, + 'site' => [ 'key' => 'static-for-now', 'name' => 'Static', 'logo' => 'node.png', @@ -158,20 +160,22 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo 'base' => 'static:1.0', 'image' => 'static:1.0', 'supports' => [System::X86, System::ARM64, System::ARMV7, System::ARMV8] - ]; + ], + default => null + }; + + if (\is_null($runtime)) { + throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); } - //todo: figure out for sites/functions - if ($isFunction) { - if (\is_null($runtime)) { - throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - } - //todo: find a better approach - $deploymentId = $isSite ? $function->getAttribute('deploymentId', '') : $function->getAttribute('deployment', ''); + $deploymentId = match($type) { + 'function' => $resource->getAttribute('deploymentId', ''), + 'site' => $resource->getAttribute('deployment', '') + }; + $deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $deploymentId)); - if ($deployment->getAttribute('resourceId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $resource->getId()) { throw new AppwriteException(AppwriteException::DEPLOYMENT_NOT_FOUND, 'Deployment not found. Create a deployment before trying to execute a function'); } @@ -190,30 +194,32 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } //todo: figure out for sites/functions - if ($isFunction) { - $permissions = $function->getAttribute('execute'); + if ($type === 'function') { + $permissions = $resource->getAttribute('execute'); if (!(\in_array('any', $permissions)) && !(\in_array('guests', $permissions))) { throw new AppwriteException(AppwriteException::USER_UNAUTHORIZED, 'To execute function using domain, execute permissions must include "any" or "guests"'); } } - $jwtExpiry = $function->getAttribute('timeout', 900); - $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); - $apiKey = $jwtObj->encode([ - 'projectId' => $project->getId(), - 'scopes' => $function->getAttribute('scopes', []) - ]); - $headers = \array_merge([], $requestHeaders); - $headers['x-appwrite-key'] = API_KEY_DYNAMIC . '_' . $apiKey; - $headers['x-appwrite-trigger'] = 'http'; $headers['x-appwrite-user-id'] = ''; - $headers['x-appwrite-user-jwt'] = ''; $headers['x-appwrite-country-code'] = ''; $headers['x-appwrite-continent-code'] = ''; $headers['x-appwrite-continent-eu'] = 'false'; + if ($type === 'function') { + $jwtExpiry = $resource->getAttribute('timeout', 900); + $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); + $apiKey = $jwtObj->encode([ + 'projectId' => $project->getId(), + 'scopes' => $resource->getAttribute('scopes', []) + ]); + $headers['x-appwrite-key'] = API_KEY_DYNAMIC . '_' . $apiKey; + $headers['x-appwrite-trigger'] = 'http'; + $headers['x-appwrite-user-jwt'] = ''; + } + $ip = $headers['x-real-ip'] ?? ''; if (!empty($ip)) { $record = $geodb->get($ip); @@ -239,8 +245,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $execution = new Document([ '$id' => $executionId, '$permissions' => [], - 'functionInternalId' => $function->getInternalId(), - 'functionId' => $function->getId(), + 'functionInternalId' => $resource->getInternalId(), + 'functionId' => $resource->getId(), 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), 'trigger' => 'http', // http / schedule / event @@ -257,9 +263,9 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ]); $queueForEvents - ->setParam('functionId', $function->getId()) + ->setParam('functionId', $resource->getId()) ->setParam('executionId', $execution->getId()) - ->setContext('function', $function); + ->setContext('function', $resource); $durationStart = \microtime(true); @@ -276,12 +282,12 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } // Shared vars - foreach ($function->getAttribute('varsProject', []) as $var) { + foreach ($resource->getAttribute('varsProject', []) as $var) { $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } // Function vars - foreach ($function->getAttribute('vars', []) as $var) { + foreach ($resource->getAttribute('vars', []) as $var) { $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } @@ -293,7 +299,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $vars = \array_merge($vars, [ 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, 'APPWRITE_FUNCTION_ID' => $resourceId, - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', @@ -320,21 +326,26 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo /** Execute function */ $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); try { - $version = $function->getAttribute('version', 'v2'); - $entrypoint = $deployment->getAttribute('entrypoint', ''); - // todo: figure out site specific settings - if ($isSite) { - $version = 'v4'; - $entrypoint = 'placeholder'; - } - $runtimeEntrypoint = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $runtime['startCommand'] . '"'; + $version = match($type) { + 'function' => $resource->getAttribute('version', 'v2'), + 'site' => 'v4' + }; + $entrypoint = match($type) { + 'function' => $deployment->getAttribute('entrypoint', ''), + 'site' => 'placeholder' // entrypoint is required in api, but not needed with site + }; + $runtimeEntrypoint = match ($version) { + 'v2' => '', + default => 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $runtime['startCommand'] . '"' + }; + $executionResponse = $executor->createExecution( projectId: $project->getId(), deploymentId: $deployment->getId(), body: \strlen($body) > 0 ? $body : null, variables: $vars, // todo: figure out timeouts for sites - timeout: $function->getAttribute('timeout', 30), + timeout: $resource->getAttribute('timeout', 30), image: $runtime['image'], source: $build->getAttribute('path', ''), entrypoint: $entrypoint, @@ -345,7 +356,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo runtimeEntrypoint: $runtimeEntrypoint, cpus: $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT, memory: $spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, - logging: $function->getAttribute('logging', true), + logging: $resource->getAttribute('logging', true), requestTimeout: 30 ); @@ -388,21 +399,22 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); - if ($isFunction) { + //todo: add metrics for sites + if ($type === 'function') { $queueForUsage ->addMetric(METRIC_EXECUTIONS, 1) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) + ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function + ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))); + ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))); } $queueForUsage ->setProject($project) ->trigger(); - if ($isFunction) { + if ($type === 'function') { $queueForFunctions ->setType(Func::TYPE_ASYNC_WRITE) ->setExecution($execution) diff --git a/app/init.php b/app/init.php index 05824389e5..162acfa821 100644 --- a/app/init.php +++ b/app/init.php @@ -277,9 +277,17 @@ const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage'; const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute'; const METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS = '{functionInternalId}.builds.compute.success'; const METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED = '{functionInternalId}.builds.compute.failed'; +const METRIC_FUNCTION_ID_BUILDS_MB_SECONDS = '{functionInternalId}.builds.mbSeconds'; +const METRIC_SITES_ID_BUILDS = 'sites.{siteInternalId}.builds'; +const METRIC_SITES_ID_BUILDS_SUCCESS = 'sites.{siteInternalId}.builds.success'; +const METRIC_SITES_ID_BUILDS_FAILED = 'sites.{siteInternalId}.builds.failed'; +const METRIC_SITES_ID_BUILDS_STORAGE = 'sites.{siteInternalId}.builds.storage'; +const METRIC_SITES_ID_BUILDS_COMPUTE = 'sites.{siteInternalId}.builds.compute'; +const METRIC_SITES_ID_BUILDS_COMPUTE_SUCCESS = 'sites.{siteInternalId}.builds.compute.success'; +const METRIC_SITES_ID_BUILDS_COMPUTE_FAILED = 'sites.{siteInternalId}.builds.compute.failed'; +const METRIC_SITES_ID_BUILDS_MB_SECONDS = 'sites.{siteInternalId}.builds.mbSeconds'; const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; -const METRIC_FUNCTION_ID_BUILDS_MB_SECONDS = '{functionInternalId}.builds.mbSeconds'; const METRIC_EXECUTIONS = 'executions'; const METRIC_EXECUTIONS_COMPUTE = 'executions.compute'; const METRIC_EXECUTIONS_MB_SECONDS = 'executions.mbSeconds'; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 312b9edcf3..4a884cd9ce 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -119,10 +119,11 @@ class Builds extends Action */ protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void { - // todo: refactor - $isFunction = $resource->getCollection() === 'functions'; - $isSite = $resource->getCollection() === 'sites'; - $foreignKey = $isFunction ? 'functionId' : 'siteId'; + $foreignKey = match($resource->getCollection()) { + 'functions' => 'functionId', + 'sites' => 'siteId', + default => throw new \Exception('Invalid resource type') + }; $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -140,30 +141,14 @@ class Builds extends Action throw new \Exception('Deployment not found', 404); } - if ($isFunction && empty($deployment->getAttribute('entrypoint', ''))) { + // todo: figure out a better way, entrypoint is not required for sites + if ($resource->getCollection() === 'functions' && empty($deployment->getAttribute('entrypoint', ''))) { throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500); } - $version = $resource->getAttribute('version', 'v2'); - - // todo: fallback for sites - if ($isSite) { - $version = 'v4'; - } + $version = $this->getVersion($resource); + $runtime = $this->getRuntime($resource, $version); $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specifications', APP_FUNCTION_SPECIFICATION_DEFAULT)]; - $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - // todo: fix for sites using frameworks - $key = $resource->getAttribute('runtime'); - $runtime = $runtimes[$key] ?? null; - - // todo: fallback for sites - if ($isSite) { - $runtime = $runtimes['node-18.0']; - } - - if (\is_null($runtime)) { - throw new \Exception('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); - } // Realtime preparation $allEvents = Event::generateEvents("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update", [ @@ -503,36 +488,6 @@ class Builds extends Action $hostname = System::getEnv('_APP_DOMAIN'); $endpoint = $protocol . '://' . $hostname . "/v1"; - //todo: ugly, but works - if ($isFunction) { - $vars = [ - ...$vars, - 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, - 'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey, - 'APPWRITE_FUNCTION_ID' => $resource->getId(), - 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_CPUS' => $cpus, - 'APPWRITE_FUNCTION_MEMORY' => $memory - ]; - } - if ($isSite) { - $vars = [ - ...$vars, - 'APPWRITE_SITE_ID' => $resource->getId(), - 'APPWRITE_SITE_NAME' => $resource->getAttribute('name'), - 'APPWRITE_SITE_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_SITE_PROJECT_ID' => $project->getId(), - 'APPWRITE_SITE_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_SITE_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_SITE_CPUS' => $cpus, - 'APPWRITE_SITE_MEMORY' => $memory - ]; - } - // Appwrite vars $vars = \array_merge($vars, [ 'APPWRITE_VERSION' => APP_VERSION_STABLE, @@ -552,13 +507,42 @@ class Builds extends Action 'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''), ]); - $command = $deployment->getAttribute('commands', ''); - - //todo: for sites use isntall and build command - if ($isSite) { - $command = 'npm ci && npm run build'; + switch ($resource->getCollection()) { + case 'functions': + $vars = [ + ...$vars, + 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, + 'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey, + 'APPWRITE_FUNCTION_ID' => $resource->getId(), + 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_FUNCTION_CPUS' => $cpus, + 'APPWRITE_FUNCTION_MEMORY' => $memory + ]; + break; + case 'sites': + $vars = [ + ...$vars, + 'APPWRITE_SITE_ID' => $resource->getId(), + 'APPWRITE_SITE_NAME' => $resource->getAttribute('name'), + 'APPWRITE_SITE_DEPLOYMENT' => $deployment->getId(), + 'APPWRITE_SITE_PROJECT_ID' => $project->getId(), + 'APPWRITE_SITE_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_SITE_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_SITE_CPUS' => $cpus, + 'APPWRITE_SITE_MEMORY' => $memory + ]; + break; } + $command = $this->getCommand( + resource: $resource, + deployment: $deployment + ); + $response = null; $err = null; @@ -570,13 +554,8 @@ class Builds extends Action $isCanceled = false; Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err, $isSite) { + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err, $version) { try { - - $version = $resource->getAttribute('version', 'v2'); - if ($isSite) { - $version = 'v4'; - } $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"'; $response = $executor->createRuntime( @@ -691,14 +670,15 @@ class Builds extends Action if ($deployment->getAttribute('activate') === true) { $resource->setAttribute('deploymentInternalId', $deployment->getInternalId()); $resource->setAttribute('live', true); - // todo: fix here how clean this is - if ($isSite) { - $resource->setAttribute('deploymentId', $deployment->getId()); - $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); - } - if ($isFunction) { - $resource->setAttribute('deployment', $deployment->getId()); - $resource = $dbForProject->updateDocument('functions', $resource->getId(), $resource); + switch ($resource->getCollection()) { + case 'functions': + $resource->setAttribute('deployment', $deployment->getId()); + $resource = $dbForProject->updateDocument('functions', $resource->getId(), $resource); + break; + case 'sites': + $resource->setAttribute('deploymentId', $deployment->getId()); + $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); + break; } } @@ -710,7 +690,7 @@ class Builds extends Action /** Update function schedule */ // Inform scheduler if function is still active - if ($isFunction) { + if ($resource->getCollection() === 'functions') { $schedule = $dbForConsole->getDocument('schedules', $resource->getAttribute('scheduleId')); $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) @@ -753,41 +733,111 @@ class Builds extends Action channels: $target['channels'], roles: $target['roles'] ); - - /** Trigger usage queue */ - if ($build->getAttribute('status') === 'ready') { - if ($isFunction) { - $queueForUsage - ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); - } - } elseif ($build->getAttribute('status') === 'failed') { - if ($isFunction) { - $queueForUsage - ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000); - } - } - if ($isFunction) { - $queueForUsage - ->addMetric(METRIC_BUILDS, 1) // per project - ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->setProject($project) - ->trigger(); - } + $this->sendUsage( + resource:$resource, + build: $build, + project: $project, + queue: $queueForUsage + ); } } + protected function sendUsage(Document $resource, Document $build, Document $project, Usage $queue): void + { + $key = match($resource->getCollection()) { + 'functions' => 'functionInternalId', + 'sites' => 'siteInternalId', + default => throw new \Exception('Invalid resource type') + }; + + $metrics = match($resource->getCollection()) { + 'functions' => [ + 'builds' => METRIC_FUNCTION_ID_BUILDS, + 'buildsSuccess' => METRIC_FUNCTION_ID_BUILDS_SUCCESS, + 'buildsFailed' => METRIC_FUNCTION_ID_BUILDS_FAILED, + 'buildsComputeSuccess' => METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS, + 'buildsComputeFailed' => METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED, + 'buildsStorage' => METRIC_FUNCTION_ID_BUILDS_STORAGE, + 'buildsCompute' => METRIC_FUNCTION_ID_BUILDS_COMPUTE, + 'buildsMbSeconds' => METRIC_FUNCTION_ID_BUILDS_MB_SECONDS + ], + 'sites' => [ + 'builds' => METRIC_SITES_ID_BUILDS, + 'buildsSuccess' => METRIC_SITES_ID_BUILDS_SUCCESS, + 'buildsFailed' => METRIC_SITES_ID_BUILDS_FAILED, + 'buildsComputeSuccess' => METRIC_SITES_ID_BUILDS_COMPUTE_SUCCESS, + 'buildsComputeFailed' => METRIC_SITES_ID_BUILDS_COMPUTE_FAILED, + 'buildsStorage' => METRIC_SITES_ID_BUILDS_STORAGE, + 'buildsCompute' => METRIC_SITES_ID_BUILDS_COMPUTE, + 'buildsMbSeconds' => METRIC_SITES_ID_BUILDS_MB_SECONDS + ] + }; + + switch ($build->getAttribute('status')) { + case 'ready': + $queue + ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsSuccess']), 1) // per function + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeSuccess']), (int)$build->getAttribute('duration', 0) * 1000); + break; + case 'failed': + $queue + ->addMetric(METRIC_BUILDS_FAILED, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsFailed']), 1) // per function + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$build->getAttribute('duration', 0) * 1000); + break; + } + + $queue + ->addMetric(METRIC_BUILDS, 1) // per project + ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['builds']), 1) // per function + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsStorage']), $build->getAttribute('size', 0)) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsCompute']), (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsMbSeconds']), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->setProject($project) + ->trigger(); + } + + protected function getRuntime(Document $resource, string $version): array + { + $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); + $key = $resource->getAttribute('runtime'); + $runtime = match ($resource->getCollection()) { + 'functions' => $runtimes[$key] ?? null, + 'sites' => $runtimes['node-18.0'] ?? null, + default => null + }; + if (\is_null($runtime)) { + throw new \Exception('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); + } + + return $runtime; + } + + protected function getVersion(Document $resource): string + { + return match ($resource->getCollection()) { + 'functions' => $resource->getAttribute('version', 'v2'), + 'sites' => 'v4', + }; + } + + protected function getCommand(Document $resource, Document $deployment): string + { + return match($resource->getCollection()) { + 'functions' => $deployment->getAttribute('command', ''), + 'sites' => implode(' && ', array_filter([ + $deployment->getAttribute('installCommand'), + $deployment->getAttribute('buildCommand') + ])) + }; + } + /** * @param string $status * @param GitHub $github From 8950f71b0addcef71393fbc810f34a6c8e5404b3 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:24:24 +0200 Subject: [PATCH 2/5] revert: comments --- app/controllers/general.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index adc2c82f49..92c787f407 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -100,8 +100,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $type = $rule->getAttribute('resourceType'); if ($type === 'function' || $type === 'site') { - // $isFunction = $type === 'function' ; - // $isSite = $type === 'site'; $resourceCollection = match($type) { 'function' => 'functions', 'site' => 'sites' From aba3a31663d1de73e3de7f9f3f9abb1a56d4c70a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:28:20 +0200 Subject: [PATCH 3/5] fix: deployment/deploymentId --- app/controllers/general.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 92c787f407..d5463935f9 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -167,8 +167,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } $deploymentId = match($type) { - 'function' => $resource->getAttribute('deploymentId', ''), - 'site' => $resource->getAttribute('deployment', '') + 'function' => $resource->getAttribute('deployment', ''), + 'site' => $resource->getAttribute('deploymentId', '') }; $deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $deploymentId)); From 5d6a8be66d425a1dfa1dcdaf439c077b7d02e8e4 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:43:12 +0200 Subject: [PATCH 4/5] fix: function command --- src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 4a884cd9ce..0e9819c8e9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -830,7 +830,7 @@ class Builds extends Action protected function getCommand(Document $resource, Document $deployment): string { return match($resource->getCollection()) { - 'functions' => $deployment->getAttribute('command', ''), + 'functions' => $deployment->getAttribute('commands', ''), 'sites' => implode(' && ', array_filter([ $deployment->getAttribute('installCommand'), $deployment->getAttribute('buildCommand') From 41484113a0b01fe880c53af34d1ae18262e3499d Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 25 Oct 2024 10:55:31 +0200 Subject: [PATCH 5/5] chore: add comments --- app/controllers/general.php | 2 ++ .../Platform/Modules/Functions/Workers/Builds.php | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index d5463935f9..378cfbfd84 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -206,6 +206,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $headers['x-appwrite-continent-code'] = ''; $headers['x-appwrite-continent-eu'] = 'false'; + //todo: check if this would work for sites if ($type === 'function') { $jwtExpiry = $resource->getAttribute('timeout', 900); $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); @@ -330,6 +331,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo }; $entrypoint = match($type) { 'function' => $deployment->getAttribute('entrypoint', ''), + //todo: check if null works 'site' => 'placeholder' // entrypoint is required in api, but not needed with site }; $runtimeEntrypoint = match ($version) { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 0e9819c8e9..4baca4f1bd 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -119,7 +119,7 @@ class Builds extends Action */ protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void { - $foreignKey = match($resource->getCollection()) { + $resourceKey = match($resource->getCollection()) { 'functions' => 'functionId', 'sites' => 'siteId', default => throw new \Exception('Invalid resource type') @@ -127,7 +127,7 @@ class Builds extends Action $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); - $log->addTag($foreignKey, $resource->getId()); + $log->addTag($resourceKey, $resource->getId()); $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); if ($resource->isEmpty()) { @@ -151,8 +151,8 @@ class Builds extends Action $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specifications', APP_FUNCTION_SPECIFICATION_DEFAULT)]; // Realtime preparation - $allEvents = Event::generateEvents("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update", [ - $foreignKey => $resource->getId(), + $allEvents = Event::generateEvents("{$resource->getCollection()}.[{$resourceKey}].deployments.[deploymentId].update", [ + $resourceKey => $resource->getId(), 'deploymentId' => $deployment->getId() ]); @@ -433,8 +433,8 @@ class Builds extends Action ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->setClass(Event::WEBHOOK_CLASS_NAME) ->setProject($project) - ->setEvent("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update") - ->setParam($foreignKey, $resource->getId()) + ->setEvent("{$resource->getCollection()}.[{$resourceKey}].deployments.[deploymentId].update") + ->setParam($resourceKey, $resource->getId()) ->setParam('deploymentId', $deployment->getId()) ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); @@ -809,7 +809,7 @@ class Builds extends Action $key = $resource->getAttribute('runtime'); $runtime = match ($resource->getCollection()) { 'functions' => $runtimes[$key] ?? null, - 'sites' => $runtimes['node-18.0'] ?? null, + 'sites' => $runtimes['node-18.0'] ?? null, //todo: fix hardcode default => null }; if (\is_null($runtime)) {