From 3c17afbaed10eef8942c46d6baac86acb4cca41d Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 25 Jan 2022 16:51:05 +0000 Subject: [PATCH 1/7] Cleanup Collections - Rename functionId to resourceId on Deployments - Add a resourceType to Deployments - Remove status, buildStdout and buildStderr from deployments - Rename dateCreated on Builds to startTime - Rename time on Builds to endTime - Added a deploymentId to Builds - Added duration to builds - Remove Vars from Build --- app/config/collections.php | 75 ++++++++--------- app/controllers/api/functions.php | 29 +++---- app/executor.php | 81 ++++++++++--------- src/Appwrite/Utopia/Response/Model/Build.php | 16 +++- .../Utopia/Response/Model/Deployment.php | 35 +++----- 5 files changed, 109 insertions(+), 127 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 7678b9c697..56883fc297 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1915,7 +1915,18 @@ $collections = [ 'filters' => [], ], [ - '$id' => 'functionId', + '$id' => 'resourceId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'resourceType', 'type' => Database::VAR_STRING, 'format' => '', 'size' => Database::LENGTH_KEY, @@ -1981,39 +1992,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => 'status', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 128, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'buildStdout', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 4096, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'buildStderr', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 4096, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], [ '$id' => 'deploy', 'type' => Database::VAR_BOOLEAN, @@ -2028,9 +2006,9 @@ $collections = [ ], 'indexes' => [ [ - '$id' => '_key_function', + '$id' => '_key_resource', 'type' => Database::INDEX_KEY, - 'attributes' => ['functionId'], + 'attributes' => ['resourceId'], 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], @@ -2050,7 +2028,7 @@ $collections = [ 'name' => 'Builds', 'attributes' => [ [ - '$id' => 'dateCreated', + '$id' => 'startTime', 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, @@ -2060,6 +2038,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'deploymentId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'runtime', 'type' => Database::VAR_STRING, @@ -2116,7 +2105,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => 'time', + '$id' => 'endTime', 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, @@ -2127,15 +2116,15 @@ $collections = [ 'filters' => [], ], [ - '$id' => 'vars', - 'type' => Database::VAR_STRING, + '$id' => 'duration', + 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => 16384, + 'size' => 0, 'signed' => true, 'required' => false, - 'default' => new stdClass(), + 'default' => null, 'array' => false, - 'filters' => ['json'], + 'filters' => [], ], [ '$id' => 'sourceType', diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index cb086ae92f..fe2273d125 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -556,7 +556,8 @@ App::post('/v1/functions/:functionId/deployments') // Remove deploy for all other deployments. $deployments = $dbForProject->find('deployments', [ new Query('deploy', Query::TYPE_EQUAL, [true]), - new Query('functionId', Query::TYPE_EQUAL, [$functionId]) + new Query('resourceId', Query::TYPE_EQUAL, [$functionId]), + new Query('resourceType', Query::TYPE_EQUAL, ['functions']) ]); foreach ($deployments as $deployment) { @@ -570,15 +571,13 @@ App::post('/v1/functions/:functionId/deployments') '$id' => $deploymentId, '$read' => ['role:all'], '$write' => ['role:all'], - 'functionId' => $function->getId(), + 'resourceId' => $function->getId(), + 'resourceType' => 'functions', 'dateCreated' => time(), 'entrypoint' => $entrypoint, 'path' => $path, 'size' => $size, 'search' => implode(' ', [$deploymentId, $entrypoint]), - 'status' => 'processing', - 'buildStdout' => '', - 'buildStderr' => '', 'deploy' => ($deploy === 'true'), ])); @@ -670,22 +669,12 @@ App::get('/v1/functions/:functionId/deployments') $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); } - $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); + $queries[] = new Query('resourceId', Query::TYPE_EQUAL, [$function->getId()]); + $queries[] = new Query('resourceType', Query::TYPE_EQUAL, ['functions']); $results = $dbForProject->find('deployments', $queries, $limit, $offset, [], [$orderType], $cursorDeployment ?? null, $cursorDirection); $sum = $dbForProject->count('deployments', $queries, APP_LIMIT_COUNT); - // Get Current Build Data - foreach ($results as &$deployment) { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - - $deployment['status'] = $build->getAttribute('status', 'processing'); - $deployment['buildStdout'] = $build->getAttribute('stdout', ''); - $deployment['buildStderr'] = $build->getAttribute('stderr', ''); - } - - var_dump($results); - $response->dynamic(new Document([ 'deployments' => $results, 'sum' => $sum, @@ -719,7 +708,7 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId') $deployment = $dbForProject->getDocument('deployments', $deploymentId); - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found', 404); } @@ -761,7 +750,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') $deployment = $dbForProject->getDocument('deployments', $deploymentId); - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found', 404); } @@ -857,7 +846,7 @@ App::post('/v1/functions/:functionId/executions') $deployment = Authorization::skip(fn() => $dbForProject->getDocument('deployments', $function->getAttribute('deployment'))); - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found. Deploy deployment before trying to execute a function', 404); } diff --git a/app/executor.php b/app/executor.php index 98ff3d9e08..104db66575 100644 --- a/app/executor.php +++ b/app/executor.php @@ -176,7 +176,7 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl // Check if runtime is active $runtime = $runtimes[$function->getAttribute('runtime', '')] ?? null; - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('deployment not found', 404); } @@ -186,6 +186,7 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl // Process environment variables $vars = \array_merge($function->getAttribute('vars', []), [ + 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint', ''), 'APPWRITE_FUNCTION_ID' => $function->getId(), 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), @@ -195,8 +196,6 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl 'INTERNAL_RUNTIME_KEY' => $secret ]); - $vars = \array_merge($vars, $build->getAttribute('vars', [])); // for gettng endpoint. - $container = 'appwrite-function-' . $deployment->getId(); if ($activeFunctions->exists($container) && !(\substr($activeFunctions->get($container)['status'], 0, 2) === 'Up')) { // Remove container if not online @@ -328,7 +327,7 @@ function execute(string $trigger, string $projectId, string $executionId, string $deployment = $database->getDocument('deployments', $function->getAttribute('deployment', '')); $build = $database->getDocument('builds', $deployment->getAttribute('buildId', '')); - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found', 404); } @@ -379,6 +378,7 @@ function execute(string $trigger, string $projectId, string $executionId, string // Process environment variables $vars = \array_merge($function->getAttribute('vars', []), [ + 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint', ''), 'APPWRITE_FUNCTION_ID' => $function->getId(), 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), @@ -393,8 +393,6 @@ function execute(string $trigger, string $projectId, string $executionId, string 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, ]); - $vars = \array_merge($vars, $build->getAttribute('vars', [])); - $container = 'appwrite-function-' . $deployment->getId(); try { @@ -405,7 +403,8 @@ function execute(string $trigger, string $projectId, string $executionId, string '$id' => $buildId, '$read' => ($userId !== '') ? ['user:' . $userId] : [], '$write' => ['role:all'], - 'dateCreated' => time(), + 'startTime' => time(), + 'deploymentId' => $deployment->getId(), 'status' => 'processing', 'outputPath' => '', 'runtime' => $function->getAttribute('runtime', ''), @@ -413,15 +412,9 @@ function execute(string $trigger, string $projectId, string $executionId, string 'sourceType' => Storage::DEVICE_LOCAL, 'stdout' => '', 'stderr' => '', - 'time' => 0, - 'vars' => [ - 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint'), - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], - 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, - ] + 'endTime' => 0, + 'duration' => 0, + 'search' => implode(' ', [$deployment->getId(), $buildId]), ])); $deployment->setAttribute('buildId', $buildId); @@ -475,6 +468,7 @@ function execute(string $trigger, string $projectId, string $executionId, string // Process environment variables $vars = \array_merge($function->getAttribute('vars', []), [ + 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint', ''), 'APPWRITE_FUNCTION_ID' => $function->getId(), 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), @@ -489,8 +483,6 @@ function execute(string $trigger, string $projectId, string $executionId, string 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId ]); - $vars = \array_merge($vars, $build->getAttribute('vars', [])); - $stdout = ''; $stderr = ''; @@ -511,7 +503,7 @@ function execute(string $trigger, string $projectId, string $executionId, string $body = \json_encode([ 'path' => '/usr/code', - 'file' => $build->getAttribute('vars', [])['ENTRYPOINT_NAME'], + 'file' => $vars['ENTRYPOINT_NAME'], 'env' => $vars, 'payload' => $data, 'timeout' => $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)) @@ -827,7 +819,8 @@ App::post('/v1/deployment') '$id' => $buildId, '$read' => (!empty($userId)) ? ['user:' . $userId] : [], '$write' => ['role:all'], - 'dateCreated' => time(), + 'startTime' => time(), + 'deploymentId' => $deploymentId, 'status' => 'processing', 'runtime' => $function->getAttribute('runtime'), 'outputPath' => '', @@ -835,15 +828,9 @@ App::post('/v1/deployment') 'sourceType' => Storage::DEVICE_LOCAL, 'stdout' => '', 'stderr' => '', - 'time' => 0, - 'vars' => [ - 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint'), - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], - 'APPWRITE_FUNCTION_PROJECT_ID' => $projectID, - ] + 'endTime' => 0, + 'duration' => 0, + 'search' => implode(' ', [$deployment->getId(), $buildId]) ])); $deployment->setAttribute('buildId', $buildId); @@ -1037,11 +1024,28 @@ function runBuildStage(string $buildId, string $projectID): Document throw new Exception('Code is not readable: ' . $build->getAttribute('source', '')); } - $vars = $build->getAttribute('vars', []); + $deployment = $database->getDocument('deployments', $build->getAttribute('deploymentId', '')); + $resourceId = $deployment->getAttribute('resourceId', ''); + $resourceType = $deployment->getAttribute('resourceType', ''); + + if (empty($resourceId)) { + throw new Exception('Invalid resource ID on build ' . $build->getId()); + } + + if (empty($resourceType)) { + throw new Exception('Invalid resource type on build' . $build->getId()); + } + + $resource = $database->getDocument($resourceType, $resourceId); + + if ($resource->isEmpty()) { + throw new Exception('Resource not found on build ' . $build->getId()); + } + + $vars = $resource->getAttribute('vars', []); // Start tracking time - $buildStart = \microtime(true); - $time = \time(); + $buildStart = \time(); $orchestration ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) @@ -1067,7 +1071,7 @@ function runBuildStage(string $buildId, string $projectID): Document workdir: '/usr/code', labels: [ 'appwrite-type' => 'function', - 'appwrite-created' => strval($time), + 'appwrite-created' => strval($buildStart), 'appwrite-runtime' => $build->getAttribute('runtime', ''), 'appwrite-project' => $projectID, 'appwrite-build' => $build->getId(), @@ -1181,19 +1185,24 @@ function runBuildStage(string $buildId, string $projectID): Document ->setAttribute('status', 'ready') ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) ->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096))) - ->setAttribute('time', $time); + ->setAttribute('startTime', $buildStart) + ->setAttribute('endTime', \time()) + ->setAttribute('duration', \time() - $buildStart); // Update build with built code attribute $build = $database->updateDocument('builds', $buildId, $build); - $buildEnd = \microtime(true); + $buildEnd = \time(); Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds'); } catch (Exception $e) { $build ->setAttribute('status', 'failed') ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) - ->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096))); + ->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096))) + ->setAttribute('startTime', $buildStart) + ->setAttribute('endTime', \microtime(true)) + ->setAttribute('duration', \microtime(true) - $buildStart); $build = $database->updateDocument('builds', $buildId, $build); diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php index fbf55a4a25..218e82d5e3 100644 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -16,12 +16,18 @@ class Build extends Model 'default' => '', 'example' => '5e5ea5c16897e', ]) - ->addRule('dateCreated', [ + ->addRule('startTime', [ 'type' => self::TYPE_INTEGER, 'description' => 'The deployment creation date in Unix timestamp.', 'default' => 0, 'example' => 1592981250, ]) + ->addRule('deploymentId', [ + 'type' => self::TYPE_STRING, + 'description' => 'The deployment that created this build', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) // Build Status // Failed - The deployment build has failed. More details can usually be found in buildStderr // Ready - The deployment build was successful and the deployment is ready to be deployed @@ -45,7 +51,13 @@ class Build extends Model 'default' => '', 'example' => '', ]) - ->addRule('time', [ + ->addRule('endTime', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The time the build was finished in Unix timestamp.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('duration', [ 'type' => self::TYPE_INTEGER, 'description' => 'The build time in seconds.', 'default' => 0, diff --git a/src/Appwrite/Utopia/Response/Model/Deployment.php b/src/Appwrite/Utopia/Response/Model/Deployment.php index db1f518c92..b3ecf02900 100644 --- a/src/Appwrite/Utopia/Response/Model/Deployment.php +++ b/src/Appwrite/Utopia/Response/Model/Deployment.php @@ -16,12 +16,18 @@ class Deployment extends Model 'default' => '', 'example' => '5e5ea5c16897e', ]) - ->addRule('functionId', [ + ->addRule('resourceId', [ 'type' => self::TYPE_STRING, - 'description' => 'Function ID.', + 'description' => 'Resource ID.', 'default' => '', 'example' => '5e5ea6g16897e', ]) + ->addRule('resourceType', [ + 'type' => self::TYPE_STRING, + 'description' => 'Resource type.', + 'default' => '', + 'example' => 'functions', + ]) ->addRule('dateCreated', [ 'type' => self::TYPE_INTEGER, 'description' => 'The deployment creation date in Unix timestamp.', @@ -30,7 +36,7 @@ class Deployment extends Model ]) ->addRule('entrypoint', [ 'type' => self::TYPE_STRING, - 'description' => 'The entrypoint file to use to execute the delpoyment code.', + 'description' => 'The entrypoint file to use to execute the deployment code.', 'default' => '', 'example' => 'enabled', ]) @@ -40,35 +46,12 @@ class Deployment extends Model 'default' => 0, 'example' => 128, ]) - // Build Status - // Failed - The deployment build has failed. More details can usually be found in buildStderr - // Ready - The deployment build was successful and the deployment is ready to be deployed - // Processing - The deployment is currently waiting to have a build triggered - // Building - The deployment is currently being built - ->addRule('status', [ - 'type' => self::TYPE_STRING, - 'description' => 'The deployment\'s current built status', - 'default' => '', - 'example' => 'ready', - ]) ->addRule('buildId', [ 'type' => self::TYPE_STRING, 'description' => 'The current build ID.', 'default' => '', 'example' => '5e5ea5c16897e', ]) - ->addRule('buildStdout', [ - 'type' => self::TYPE_STRING, - 'description' => 'The stdout of the build.', - 'default' => '', - 'example' => '', - ]) - ->addRule('buildStderr', [ - 'type' => self::TYPE_STRING, - 'description' => 'The stderr of the build.', - 'default' => '', - 'example' => '', - ]) ->addRule('deploy', [ 'type' => self::TYPE_BOOLEAN, 'description' => 'Whether the deployment should be automatically deployed.', From d429aa180f72331f1b613e353fbe8ede6bfd380a Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 26 Jan 2022 10:09:11 +0000 Subject: [PATCH 2/7] Fix Build Structure in Executor --- app/executor.php | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/app/executor.php b/app/executor.php index 26711226fb..3c92dc33b0 100644 --- a/app/executor.php +++ b/app/executor.php @@ -673,6 +673,10 @@ function runBuildStage(string $buildId, string $projectID): Document // Check if build has already been run $build = $database->getDocument('builds', $buildId); + + // Start tracking time + $buildStart = \time(); + try { // If we already have a built package ready there is no need to rebuild. if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) { @@ -728,9 +732,6 @@ function runBuildStage(string $buildId, string $projectID): Document $vars = $build->getAttribute('vars', []); - // Start tracking time - $buildStart = \time(); - $orchestration ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) @@ -1084,23 +1085,17 @@ App::post('/v1/deployment') '$id' => $buildId, '$read' => (!empty($userId)) ? ['user:' . $userId] : [], '$write' => ['role:all'], - 'dateCreated' => time(), + 'startTime' => time(), 'status' => 'processing', + 'deploymentId' => $deploymentId, 'runtime' => $function->getAttribute('runtime'), 'outputPath' => '', 'source' => $deployment->getAttribute('path'), 'sourceType' => Storage::DEVICE_LOCAL, 'stdout' => '', 'stderr' => '', - 'time' => 0, - 'vars' => [ - 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint'), - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], - 'APPWRITE_FUNCTION_PROJECT_ID' => $projectID, - ] + 'endTime' => 0, + 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); From 18e142089911c6ba434666674aecb43a98498711 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 26 Jan 2022 10:10:29 +0000 Subject: [PATCH 3/7] Update executor.php --- app/executor.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/executor.php b/app/executor.php index 3c92dc33b0..cdc44cb569 100644 --- a/app/executor.php +++ b/app/executor.php @@ -1095,7 +1095,8 @@ App::post('/v1/deployment') 'stdout' => '', 'stderr' => '', 'endTime' => 0, - 'duration' => 0 + 'duration' => 0, + 'search' => implode(' ', [$deployment->getId(), $buildId]) ])); $deployment->setAttribute('buildId', $buildId); From f3e97e41334f20d4cae8e695103ff6ddf53681f7 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 26 Jan 2022 10:49:02 +0000 Subject: [PATCH 4/7] Fix environment variables during build --- app/config/collections.php | 44 +++++++++++++++++++------------------- app/executor.php | 20 ++++++++++++++++- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 56883fc297..befcc5e06a 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2038,6 +2038,28 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'endTime', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'duration', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'deploymentId', 'type' => Database::VAR_STRING, @@ -2104,28 +2126,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => 'endTime', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'duration', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ '$id' => 'sourceType', 'type' => Database::VAR_STRING, diff --git a/app/executor.php b/app/executor.php index cdc44cb569..f0ca068a23 100644 --- a/app/executor.php +++ b/app/executor.php @@ -730,7 +730,25 @@ function runBuildStage(string $buildId, string $projectID): Document throw new Exception('Code is not readable: ' . $build->getAttribute('source', '')); } - $vars = $build->getAttribute('vars', []); + $deployment = $database->getDocument('deployments', $build->getAttribute('deploymentId', '')); + $resourceId = $deployment->getAttribute('resourceId', ''); + $resourceType = $deployment->getAttribute('resourceType', ''); + + if (empty($resourceId)) { + throw new Exception('Invalid resource ID on build ' . $build->getId()); + } + + if (empty($resourceType)) { + throw new Exception('Invalid resource type on build' . $build->getId()); + } + + $resource = $database->getDocument($resourceType, $resourceId); + + if ($resource->isEmpty()) { + throw new Exception('Resource not found on build ' . $build->getId()); + } + + $vars = $resource->getAttribute('vars', []); $orchestration ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) From 8e639d4f5f85526928596e5a2c921d5a83afdedf Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 31 Jan 2022 11:29:31 +0000 Subject: [PATCH 5/7] Fix UI --- app/controllers/api/functions.php | 7 +++++++ .../Utopia/Response/Model/Deployment.php | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 47d6c1bd00..ddfe50cda0 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -675,6 +675,13 @@ App::get('/v1/functions/:functionId/deployments') $results = $dbForProject->find('deployments', $queries, $limit, $offset, [], [$orderType], $cursorDeployment ?? null, $cursorDirection); $sum = $dbForProject->count('deployments', $queries, APP_LIMIT_COUNT); + foreach ($results as $result) { + $build = $dbForProject->getDocument('builds', $result->getAttribute('buildId')); + $result->setAttribute('status', $build->getAttribute('status', 'pending')); + $result->setAttribute('buildStderr', $build->getAttribute('stderr', '')); + $result->setAttribute('buildStdout', $build->getAttribute('stdout', '')); + } + $response->dynamic(new Document([ 'deployments' => $results, 'sum' => $sum, diff --git a/src/Appwrite/Utopia/Response/Model/Deployment.php b/src/Appwrite/Utopia/Response/Model/Deployment.php index b3ecf02900..28a01ad6c0 100644 --- a/src/Appwrite/Utopia/Response/Model/Deployment.php +++ b/src/Appwrite/Utopia/Response/Model/Deployment.php @@ -58,6 +58,24 @@ class Deployment extends Model 'default' => false, 'example' => true, ]) + ->addRule('status', [ + 'type' => self::TYPE_STRING, + 'description' => 'The deployment status.', + 'default' => '', + 'example' => 'enabled', + ]) + ->addRule('buildStdout', [ + 'type' => self::TYPE_STRING, + 'description' => 'The build stdout.', + 'default' => '', + 'example' => 'enabled', + ]) + ->addRule('buildStderr', [ + 'type' => self::TYPE_STRING, + 'description' => 'The build stderr.', + 'default' => '', + 'example' => 'enabled', + ]) ; } From d2ec314ef0733b1803d53a7f5fbc51d21900d5d6 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 31 Jan 2022 11:36:22 +0000 Subject: [PATCH 6/7] Remove Search and add resourceType to indexes --- app/config/collections.php | 20 ++++----- app/controllers/api/functions.php | 71 ------------------------------- app/executor.php | 6 +-- 3 files changed, 10 insertions(+), 87 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index befcc5e06a..b8b0afdea6 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2012,6 +2012,13 @@ $collections = [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_resource_type', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceType'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], [ '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, @@ -2147,18 +2154,7 @@ $collections = [ 'default' => '', 'array' => false, 'filters' => [], - ], - [ - '$id' => 'search', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], + ] ], 'indexes' => [ [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index ddfe50cda0..db0950c43b 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1052,77 +1052,6 @@ App::get('/v1/functions/:functionId/executions/:executionId') $response->dynamic($execution, Response::MODEL_EXECUTION); }); -App::get('/v1/builds') -->groups(['api', 'functions']) -->desc('List Builds') -->label('scope', 'functions.read') -->label('sdk.auth', [APP_AUTH_TYPE_KEY]) -->label('sdk.namespace', 'functions') -->label('sdk.method', 'builds') -->label('sdk.description', '/docs/references/functions/list-builds.md') -->label('sdk.response.code', Response::STATUS_CODE_OK) -->label('sdk.response.type', Response::CONTENT_TYPE_JSON) -->label('sdk.response.model', Response::MODEL_BUILD_LIST) -->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) -->param('limit', 25, new Range(0, 100), 'Maximum number of builds to return in response. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) -->param('offset', 0, new Range(0, APP_LIMIT_COUNT), 'Offset value. The default value is 0. Use this value to manage pagination. [learn more about pagination](https://appwrite.io/docs/pagination)', true) -->param('cursor', '', new UID(), 'ID of the function used as the starting point for the query, excluding the function itself. Should be used for efficient pagination when working with large sets of data. [learn more about pagination](https://appwrite.io/docs/pagination)', true) -->param('cursorDirection', Database::CURSOR_AFTER, new WhiteList([Database::CURSOR_AFTER, Database::CURSOR_BEFORE]), 'Direction of the cursor.', true) -->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) -->inject('response') -->inject('dbForProject') -->action(function ($search, $limit, $offset, $cursor, $cursorDirection, $orderType, $response, $dbForProject) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Database $dbForProject */ - - if (!empty($cursor)) { - $cursorFunction = $dbForProject->getDocument('builds', $cursor); - - if ($cursorFunction->isEmpty()) { - throw new Exception("Build '{$cursor}' for the 'cursor' value not found.", 400); - } - } - - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); - } - - $response->dynamic(new Document([ - 'builds' => $dbForProject->find('builds', $queries, $limit, $offset, [], [$orderType], $cursorFunction ?? null, $cursorDirection), - 'sum' => $dbForProject->count('builds', $queries, APP_LIMIT_COUNT), - ]), Response::MODEL_BUILD_LIST); -}); - -App::get('/v1/builds/:buildId') - ->groups(['api', 'functions']) - ->desc('Get Build') - ->label('scope', 'execution.read') - ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) - ->label('sdk.namespace', 'functions') - ->label('sdk.method', 'getBuild') - ->label('sdk.description', '/docs/references/functions/get-build.md') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_BUILD) - ->param('buildId', '', new UID(), 'Build unique ID.') - ->inject('response') - ->inject('dbForProject') - ->action(function ($buildId, $response, $dbForProject) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Database $dbForProject */ - - $build = Authorization::skip(function () use ($dbForProject, $buildId) { - return $dbForProject->getDocument('builds', $buildId); - }); - - if ($build->isEmpty()) { - throw new Exception('Build not found', 404); - } - - $response->dynamic($build, Response::MODEL_BUILD); - }); App::post('/v1/builds/:buildId') ->groups(['api', 'functions']) ->desc('Retry Build') diff --git a/app/executor.php b/app/executor.php index f0ca068a23..8d9d593f1a 100644 --- a/app/executor.php +++ b/app/executor.php @@ -420,8 +420,7 @@ function execute(string $trigger, string $projectId, string $executionId, string 'stdout' => '', 'stderr' => '', 'endTime' => 0, - 'duration' => 0, - 'search' => implode(' ', [$deployment->getId(), $buildId]), + 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); @@ -1113,8 +1112,7 @@ App::post('/v1/deployment') 'stdout' => '', 'stderr' => '', 'endTime' => 0, - 'duration' => 0, - 'search' => implode(' ', [$deployment->getId(), $buildId]) + 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); From 83e9e9f0164c22c458f6257ca4edeaa9fe5a15bb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 31 Jan 2022 15:57:55 +0400 Subject: [PATCH 7/7] feat: remove search index and updates tests --- app/config/collections.php | 9 +----- .../Functions/FunctionsCustomServerTest.php | 32 ------------------- 2 files changed, 1 insertion(+), 40 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index b8b0afdea6..15471c8c77 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2163,14 +2163,7 @@ $collections = [ 'attributes' => ['status'], 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_search', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], + ] ], ], diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index a9e9096fe6..ce220806bb 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -331,38 +331,6 @@ class FunctionsCustomServerTest extends Scope return $data; } - /** - * @depends testCreateDeployment - */ - public function testListBuild(array $data):array - { - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_GET, '/builds', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(1, $response['body']['sum']); - $this->assertEquals($response['body']['builds'][0]['status'], 'ready'); - - /** - * Check Queries work - */ - $responseQuery = $this->client->call(Client::METHOD_GET, '/builds?status=status.equal(\"ready\")', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); - - $this->assertEquals(200, $responseQuery['headers']['status-code']); - $this->assertEquals(1, $responseQuery['body']['sum']); - $this->assertEquals($responseQuery['body']['builds'][0]['status'], 'ready'); - - return $data; - } - /** * @depends testCreateDeployment */