From a82562c16e6fcca5ee5bfdaf2d374a32ddc79947 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 03:36:30 +0400 Subject: [PATCH 01/21] feat: add deploymentId property to builds collection --- app/config/collections.php | 15 ++++++++-- app/controllers/api/functions.php | 46 +++++++++++++++---------------- app/executor.php | 1 + app/init.php | 1 + app/workers/builds.php | 1 + app/workers/deletes.php | 23 ++++++++++++++-- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 7678b9c697..418fdfa3c9 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2049,6 +2049,17 @@ $collections = [ '$id' => 'builds', 'name' => 'Builds', 'attributes' => [ + [ + '$id' => 'deploymentId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'dateCreated', 'type' => Database::VAR_INTEGER, @@ -2173,9 +2184,9 @@ $collections = [ ], 'indexes' => [ [ - '$id' => '_key_status', + '$id' => '_key_deployment', 'type' => Database::INDEX_KEY, - 'attributes' => ['status'], + 'attributes' => ['deploymentId'], 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 47cae1cec3..aa341d62bf 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -431,34 +431,34 @@ App::delete('/v1/functions/:functionId') $function = $dbForProject->getDocument('functions', $functionId); - // Request executor to delete deployment containers - $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/functions/$functionId"); - \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - \curl_setopt($ch, CURLOPT_TIMEOUT, 900); - \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - \curl_setopt($ch, CURLOPT_HTTPHEADER, [ - 'Content-Type: application/json', - 'x-appwrite-project: '.$project->getId(), - 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') - ]); + // // Request executor to delete deployment containers + // $ch = \curl_init(); + // \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); + // \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/functions/$functionId"); + // \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + // \curl_setopt($ch, CURLOPT_TIMEOUT, 900); + // \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + // \curl_setopt($ch, CURLOPT_HTTPHEADER, [ + // 'Content-Type: application/json', + // 'x-appwrite-project: '.$project->getId(), + // 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') + // ]); - $executorResponse = \curl_exec($ch); + // $executorResponse = \curl_exec($ch); - $error = \curl_error($ch); + // $error = \curl_error($ch); - if (!empty($error)) { - throw new Exception('Executor Cleanup Error: ' . $error, 500); - } + // if (!empty($error)) { + // throw new Exception('Executor Cleanup Error: ' . $error, 500); + // } - // Check status code - $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); - if (200 !== $statusCode) { - throw new Exception('Executor error: ' . $executorResponse, $statusCode); - } + // // Check status code + // $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); + // if (200 !== $statusCode) { + // throw new Exception('Executor error: ' . $executorResponse, $statusCode); + // } - \curl_close($ch); + // \curl_close($ch); if ($function->isEmpty()) { throw new Exception('Function not found', 404); diff --git a/app/executor.php b/app/executor.php index 3bd6eb5f4b..2d3cbd7ab3 100644 --- a/app/executor.php +++ b/app/executor.php @@ -413,6 +413,7 @@ function execute(string $trigger, string $projectId, string $executionId, string '$id' => $buildId, '$read' => ($userId !== '') ? ['user:' . $userId] : [], '$write' => [], + 'deploymentId' => $deployment->getId(), 'dateCreated' => time(), 'status' => 'processing', 'outputPath' => '', diff --git a/app/init.php b/app/init.php index fe330815e8..3c22aa3aef 100644 --- a/app/init.php +++ b/app/init.php @@ -100,6 +100,7 @@ const DELETE_TYPE_DOCUMENT = 'document'; const DELETE_TYPE_COLLECTIONS = 'collections'; const DELETE_TYPE_PROJECTS = 'projects'; const DELETE_TYPE_FUNCTIONS = 'functions'; +const DELETE_TYPE_DEPLOYMENTS = 'deployments'; const DELETE_TYPE_USERS = 'users'; const DELETE_TYPE_TEAMS= 'teams'; const DELETE_TYPE_EXECUTIONS = 'executions'; diff --git a/app/workers/builds.php b/app/workers/builds.php index ac20e0031a..0e529249d9 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -118,6 +118,7 @@ class BuildsV1 extends Worker '$id' => $buildId, '$read' => [], '$write' => [], + 'deploymentId' => $deploymentId, 'dateCreated' => time(), 'status' => 'processing', 'runtime' => $function->getAttribute('runtime'), diff --git a/app/workers/deletes.php b/app/workers/deletes.php index bc7721d550..4795b94412 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -50,6 +50,9 @@ class DeletesV1 extends Worker case DELETE_TYPE_FUNCTIONS: $this->deleteFunction($document, $projectId); break; + case DELETE_TYPE_DEPLOYMENTS: + $this->deleteDeployment($document, $projectId); + break; case DELETE_TYPE_USERS: $this->deleteUser($document, $projectId); break; @@ -313,18 +316,34 @@ class DeletesV1 extends Worker $this->deleteByGroup('deployments', [ new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject, function (Document $document) use ($device) { - - if ($device->delete($document->getAttribute('path', ''))) { + if ($device->delete($document->getAttribute('path', ''), true)) { Console::success('Delete code deployment: ' . $document->getAttribute('path', '')); } else { Console::error('Failed to delete code deployment: ' . $document->getAttribute('path', '')); } }); + // Delete builds + $this->deleteByGroup('builds', [ + new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) + ], $dbForProject, function (Document $document) use ($device) { + if ($device->delete($document->getAttribute('path', ''), true)) { + Console::success('Delete code deployment: ' . $document->getAttribute('path', '')); + } else { + Console::error('Failed to delete code deployment: ' . $document->getAttribute('path', '')); + } + }); + + // Delete build files + // Delete Executions $this->deleteByGroup('executions', [ new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject); + + + // Delete deployment files + } From 1d5b6b7fd9bbab700104835a4abfa9b51a5f9844 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 04:02:44 +0400 Subject: [PATCH 02/21] feat: delete built files in deletes worker --- app/workers/deletes.php | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 4795b94412..57c6f7a6db 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -311,39 +311,36 @@ class DeletesV1 extends Worker { $dbForProject = $this->getProjectDB($projectId); $device = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); - + $deploymentIds = []; + // Delete Deployments $this->deleteByGroup('deployments', [ new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) - ], $dbForProject, function (Document $document) use ($device) { + ], $dbForProject, function (Document $document) use ($device, &$deploymentIds) { + $deploymentIds[] = $document->getId(); if ($device->delete($document->getAttribute('path', ''), true)) { - Console::success('Delete code deployment: ' . $document->getAttribute('path', '')); + Console::success('Delete deployment files: ' . $document->getAttribute('path', '')); } else { - Console::error('Failed to delete code deployment: ' . $document->getAttribute('path', '')); + Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); } }); // Delete builds $this->deleteByGroup('builds', [ - new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) + new Query('deploymentId', Query::TYPE_EQUAL, $deploymentIds) ], $dbForProject, function (Document $document) use ($device) { - if ($device->delete($document->getAttribute('path', ''), true)) { - Console::success('Delete code deployment: ' . $document->getAttribute('path', '')); + if ($device->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); } else { - Console::error('Failed to delete code deployment: ' . $document->getAttribute('path', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); } }); - // Delete build files - // Delete Executions $this->deleteByGroup('executions', [ new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject); - - // Delete deployment files - } @@ -388,9 +385,7 @@ class DeletesV1 extends Worker $executionStart = \microtime(true); while ($sum === $limit) { - Authorization::disable(); - $projects = $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit)); - Authorization::reset(); + $projects = Authorization::skip(fn() => $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit))); $chunk++; @@ -429,11 +424,7 @@ class DeletesV1 extends Worker while ($sum === $limit) { $chunk++; - Authorization::disable(); - - $results = $database->find($collection, $queries, $limit, 0); - - Authorization::reset(); + $results = Authorization::skip(fn() => $database->find($collection, $queries, $limit, 0)); $sum = count($results); From 80b99ea146c814288a2d32bbbc7eb2886df0d7c6 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 04:17:59 +0400 Subject: [PATCH 03/21] feat: added build mount to deletes worker --- app/workers/deletes.php | 11 ++++++----- docker-compose.yml | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 57c6f7a6db..37fe8db3af 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -310,15 +310,16 @@ class DeletesV1 extends Worker protected function deleteFunction(Document $document, string $projectId): void { $dbForProject = $this->getProjectDB($projectId); - $device = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); + $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); + $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); $deploymentIds = []; // Delete Deployments $this->deleteByGroup('deployments', [ new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) - ], $dbForProject, function (Document $document) use ($device, &$deploymentIds) { + ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { $deploymentIds[] = $document->getId(); - if ($device->delete($document->getAttribute('path', ''), true)) { + if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { Console::success('Delete deployment files: ' . $document->getAttribute('path', '')); } else { Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); @@ -328,8 +329,8 @@ class DeletesV1 extends Worker // Delete builds $this->deleteByGroup('builds', [ new Query('deploymentId', Query::TYPE_EQUAL, $deploymentIds) - ], $dbForProject, function (Document $document) use ($device) { - if ($device->delete($document->getAttribute('outputPath', ''), true)) { + ], $dbForProject, function (Document $document) use ($storageBuilds) { + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); } else { Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); diff --git a/docker-compose.yml b/docker-compose.yml index 1ad9d203fe..3b4ea3a636 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,6 +66,7 @@ services: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw - appwrite-functions:/storage/functions:rw + - appwrite-builds:/storage/builds:rw - ./phpunit.xml:/usr/src/code/phpunit.xml - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests @@ -238,6 +239,7 @@ services: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - appwrite-functions:/storage/functions:rw + - appwrite-builds:/storage/builds:rw - appwrite-certificates:/storage/certificates:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src @@ -398,6 +400,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-functions:/storage/functions:rw + - appwrite-builds:/storage/builds:rw - /tmp:/tmp:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src @@ -697,6 +700,7 @@ volumes: appwrite-uploads: appwrite-certificates: appwrite-functions: + appwrite-builds: appwrite-influxdb: appwrite-config: appwrite-executor: From 700c2fff2beb2a6c4b7854a0bce7af022f02a3ad Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 04:23:49 +0400 Subject: [PATCH 04/21] feat: remove builds mount from trafeik --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3b4ea3a636..68cc05c64d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,7 +66,6 @@ services: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw - appwrite-functions:/storage/functions:rw - - appwrite-builds:/storage/builds:rw - ./phpunit.xml:/usr/src/code/phpunit.xml - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests From 608eff458c39b55c987e533125d8e931b89ed2b1 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 04:25:28 +0400 Subject: [PATCH 05/21] feat: refactor --- app/workers/deletes.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 37fe8db3af..7ac9f13fe9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -312,9 +312,9 @@ class DeletesV1 extends Worker $dbForProject = $this->getProjectDB($projectId); $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); - $deploymentIds = []; - + // Delete Deployments + $deploymentIds = []; $this->deleteByGroup('deployments', [ new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { From dbcf479cd943c7ecc1cfbbde8f6899443b7e6be6 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 04:26:19 +0400 Subject: [PATCH 06/21] feat: remove curl request from DELETE /function --- app/controllers/api/functions.php | 29 ----------------------------- app/workers/deletes.php | 2 ++ 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index aa341d62bf..7f666b628e 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -431,35 +431,6 @@ App::delete('/v1/functions/:functionId') $function = $dbForProject->getDocument('functions', $functionId); - // // Request executor to delete deployment containers - // $ch = \curl_init(); - // \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); - // \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/functions/$functionId"); - // \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - // \curl_setopt($ch, CURLOPT_TIMEOUT, 900); - // \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - // \curl_setopt($ch, CURLOPT_HTTPHEADER, [ - // 'Content-Type: application/json', - // 'x-appwrite-project: '.$project->getId(), - // 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') - // ]); - - // $executorResponse = \curl_exec($ch); - - // $error = \curl_error($ch); - - // if (!empty($error)) { - // throw new Exception('Executor Cleanup Error: ' . $error, 500); - // } - - // // Check status code - // $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); - // if (200 !== $statusCode) { - // throw new Exception('Executor error: ' . $executorResponse, $statusCode); - // } - - // \curl_close($ch); - if ($function->isEmpty()) { throw new Exception('Function not found', 404); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 7ac9f13fe9..dc66edffcd 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -342,6 +342,8 @@ class DeletesV1 extends Worker new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject); + // Send delete request to executor + } From 5125adc16bf705af9a5c4fdd234e872c8dbfcbde Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 05:21:11 +0400 Subject: [PATCH 07/21] feat: use coroutines for deleting within the executor --- app/executor.php | 64 ++++++++++++++++++++++------------------- app/workers/deletes.php | 45 +++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 36 deletions(-) diff --git a/app/executor.php b/app/executor.php index 2d3cbd7ab3..8cab18855b 100644 --- a/app/executor.php +++ b/app/executor.php @@ -947,58 +947,62 @@ App::post('/v1/functions/:functionId/executions') App::delete('/v1/functions/:functionId') ->desc('Delete a function') ->param('functionId', '', new UID()) + ->inject('projectId') ->inject('response') ->inject('dbForProject') ->action( - function (string $functionId, Response $response, Database $dbForProject) use ($orchestrationPool) { - try { - /** @var Orchestration $orchestration */ - $orchestration = $orchestrationPool->get(); + function (string $functionId, string $projectId, Response $response, Database $dbForProject) use ($orchestrationPool) { - // Get function document - $function = $dbForProject->getDocument('functions', $functionId); + $results = $dbForProject->find('deployments', [new Query('functionId', Query::TYPE_EQUAL, [$functionId])], 999); - // Check if function exists - if ($function->isEmpty()) { - throw new Exception('Function not found', 404); - } + // If amount is 0 then we simply return true + if (count($results) === 0) { + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->send(); + } - $results = $dbForProject->find('deployments', [new Query('functionId', Query::TYPE_EQUAL, [$functionId])], 999); + Console::info("Deleting $results count"); - // If amount is 0 then we simply return true - if (count($results) === 0) { - return $response->json(['success' => true]); - } - - // Delete the containers of all deployments - foreach ($results as $deployment) { + // Delete the containers of all deployments + // TODO : @christy Delete all build containers as well. Not just the latest one, + global $register; + foreach ($results as $deployment) { + go(function () use ($orchestrationPool, $deployment, $register, $projectId) { try { + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + $cache = new Cache(new RedisCache($redis)); + $dbForProject = new Database(new MariaDB($db), $cache); + $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $dbForProject->setNamespace('_project_' . $projectId); + + $orchestration = $orchestrationPool->get(); // Remove any ongoing builds if ($deployment->getAttribute('buildId')) { $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - + if ($build->getAttribute('status') === 'building') { // Remove the build $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); Console::info('Removed build for deployment ' . $deployment['$id']); } } - + $orchestration->remove('appwrite-function-' . $deployment['$id'], true); Console::info('Removed container for deployment ' . $deployment['$id']); - } catch (Exception $e) { + } catch (Throwable $th) { // Do nothing, we don't care that much if it fails + Console::error($th->getMessage()); + } finally { + $orchestrationPool->put($orchestration); } - } - - return $response->json(['success' => true]); - } catch (Exception $e) { - logError($e, "cleanupFunction"); - $orchestrationPool->put($orchestration); - - return $response->json(['error' => $e->getMessage()]); + }); } - $orchestrationPool->put($orchestration); + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->send(); } ); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index dc66edffcd..057926f41d 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -8,6 +8,7 @@ use Appwrite\Resque\Worker; use Utopia\Storage\Device\Local; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Audit\Audit; @@ -310,10 +311,41 @@ class DeletesV1 extends Worker protected function deleteFunction(Document $document, string $projectId): void { $dbForProject = $this->getProjectDB($projectId); - $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); - $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); - // Delete Deployments + /** + * Request executor to delete all deployment containers + */ + try { + $ch = \curl_init(); + \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/functions/{$document->getId()}"); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'x-appwrite-project: '. $projectId, + 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') + ]); + + $executorResponse = \curl_exec($ch); + $error = \curl_error($ch); + if (!empty($error)) { + throw new Exception($error, 500); + } + + $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($statusCode >= 400) { + throw new Exception('Executor error: ' . $executorResponse, $statusCode); + } + + \curl_close($ch); + } catch (Throwable $th) { + Console::error($th->getMessage()); + } + + /** + * Delete Deployments + */ + $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); $deploymentIds = []; $this->deleteByGroup('deployments', [ new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) @@ -326,7 +358,10 @@ class DeletesV1 extends Worker } }); - // Delete builds + /** + * Delete builds + */ + $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); $this->deleteByGroup('builds', [ new Query('deploymentId', Query::TYPE_EQUAL, $deploymentIds) ], $dbForProject, function (Document $document) use ($storageBuilds) { @@ -342,8 +377,6 @@ class DeletesV1 extends Worker new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject); - // Send delete request to executor - } From 09de5ed256dbaba7f0a4652ff759e38f1a5d6ca2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 05:29:20 +0400 Subject: [PATCH 08/21] feat: delete deployment deletion to the worker --- app/controllers/api/functions.php | 53 +++++---------------- app/workers/deletes.php | 76 +++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 41 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7f666b628e..43c9257880 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -685,64 +685,30 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ->inject('response') ->inject('dbForProject') ->inject('usage') + ->inject('deletes') ->inject('project') - ->action(function ($functionId, $deploymentId, $response, $dbForProject, $usage, $project) { + ->action(function ($functionId, $deploymentId, $response, $dbForProject, $usage, $deletes, $project) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $usage */ /** @var Utopia\Database\Document $project */ $function = $dbForProject->getDocument('functions', $functionId); - if ($function->isEmpty()) { throw new Exception('Function not found', 404); } $deployment = $dbForProject->getDocument('deployments', $deploymentId); + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found', 404); + } if ($deployment->getAttribute('functionId') !== $function->getId()) { throw new Exception('Deployment not found', 404); } - if ($deployment->isEmpty()) { - throw new Exception('deployment not found', 404); - } - - // Request executor to delete deployment containers - $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/deployments/$deploymentId"); - \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - \curl_setopt($ch, CURLOPT_TIMEOUT, 900); - \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - \curl_setopt($ch, CURLOPT_HTTPHEADER, [ - 'Content-Type: application/json', - 'x-appwrite-project: '.$project->getId(), - 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') - ]); - - $executorResponse = \curl_exec($ch); - - $error = \curl_error($ch); - - if (!empty($error)) { - throw new Exception('Executor Cleanup error: ' . $error, 500); - } - - // Check status code - $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); - if (200 !== $statusCode) { - throw new Exception('Executor error: ' . $executorResponse, $statusCode); - } - - \curl_close($ch); - - $device = Storage::getDevice('functions'); - - if ($device->delete($deployment->getAttribute('path', ''))) { - if (!$dbForProject->deleteDocument('deployments', $deployment->getId())) { - throw new Exception('Failed to remove deployment from DB', 500); - } + if (!$dbForProject->deleteDocument('deployments', $deployment->getId())) { + throw new Exception('Failed to remove deployment from DB', 500); } if($function->getAttribute('deployment') === $deployment->getId()) { // Reset function deployment @@ -755,6 +721,11 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ->setParam('storage', $deployment->getAttribute('size', 0) * -1) ; + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $deployment) + ; + $response->noContent(); }); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 057926f41d..12999d432e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -379,6 +379,82 @@ class DeletesV1 extends Worker } + /** + * @param Document $document deployment document + * @param string $projectId + */ + protected function deleteDeployment(Document $document, string $projectId): void + { + $dbForProject = $this->getProjectDB($projectId); + + /** + * Request executor to delete the deployment containers + */ + try { + $ch = \curl_init(); + \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); + // TODO: Implement coroutines. + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/deployments/{$document->getId()}"); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'x-appwrite-project: '. $projectId, + 'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '') + ]); + + $executorResponse = \curl_exec($ch); + $error = \curl_error($ch); + if (!empty($error)) { + throw new Exception($error, 500); + } + + $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($statusCode >= 400) { + throw new Exception('Executor error: ' . $executorResponse, $statusCode); + } + + \curl_close($ch); + } catch (Throwable $th) { + Console::error($th->getMessage()); + } + + /** + * Delete Deployments + */ + $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); + $deploymentIds = []; + $this->deleteByGroup('deployments', [ + new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) + ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { + $deploymentIds[] = $document->getId(); + if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { + Console::success('Delete deployment files: ' . $document->getAttribute('path', '')); + } else { + Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); + } + }); + + /** + * Delete builds + */ + $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); + $this->deleteByGroup('builds', [ + new Query('deploymentId', Query::TYPE_EQUAL, $deploymentIds) + ], $dbForProject, function (Document $document) use ($storageBuilds) { + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + } else { + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + } + }); + + // Delete Executions + $this->deleteByGroup('executions', [ + new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) + ], $dbForProject); + + } + /** * @param Document $document to be deleted From 8e10d5cad9caf1d4269d00fdfcea743312c9e1b2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 05:29:36 +0400 Subject: [PATCH 09/21] feat: remove unused injection --- app/controllers/api/functions.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 43c9257880..5f4feeda87 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -686,8 +686,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ->inject('dbForProject') ->inject('usage') ->inject('deletes') - ->inject('project') - ->action(function ($functionId, $deploymentId, $response, $dbForProject, $usage, $deletes, $project) { + ->action(function ($functionId, $deploymentId, $response, $dbForProject, $usage, $deletes) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $usage */ From 6e44812d9925cb9355cbb5ec3147a6d8404d695b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 05:30:24 +0400 Subject: [PATCH 10/21] feat: remove unused injection --- app/controllers/api/functions.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5f4feeda87..31a6c593a3 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -422,12 +422,10 @@ App::delete('/v1/functions/:functionId') ->inject('response') ->inject('dbForProject') ->inject('deletes') - ->inject('project') - ->action(function ($functionId, $response, $dbForProject, $deletes, $project) { + ->action(function ($functionId, $response, $dbForProject, $deletes) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $deletes */ - /** @var Utopia\Database\Document $project */ $function = $dbForProject->getDocument('functions', $functionId); @@ -690,7 +688,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $usage */ - /** @var Utopia\Database\Document $project */ + /** @var Appwrite\Event\Event $deletes */ $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { From f4616a6832dde62a4029b500d0f2ab61e9f5cb70 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 05:32:56 +0400 Subject: [PATCH 11/21] feat: added deleteDeployment to deletes worker --- app/workers/deletes.php | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 12999d432e..8ae2bbe41c 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -422,24 +422,18 @@ class DeletesV1 extends Worker * Delete Deployments */ $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); - $deploymentIds = []; - $this->deleteByGroup('deployments', [ - new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) - ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { - $deploymentIds[] = $document->getId(); - if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { - Console::success('Delete deployment files: ' . $document->getAttribute('path', '')); - } else { - Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); - } - }); + if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { + Console::success('Delete deployment files: ' . $document->getAttribute('path', '')); + } else { + Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); + } /** * Delete builds */ $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); $this->deleteByGroup('builds', [ - new Query('deploymentId', Query::TYPE_EQUAL, $deploymentIds) + new Query('deploymentId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject, function (Document $document) use ($storageBuilds) { if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); @@ -448,11 +442,6 @@ class DeletesV1 extends Worker } }); - // Delete Executions - $this->deleteByGroup('executions', [ - new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) - ], $dbForProject); - } From 3418c06bc829b0ae2bd3dd30475734457061e0de Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 05:40:53 +0400 Subject: [PATCH 12/21] feat: added deleteDeployment to executor --- app/executor.php | 54 +++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/app/executor.php b/app/executor.php index 8cab18855b..1e4ceee126 100644 --- a/app/executor.php +++ b/app/executor.php @@ -962,7 +962,7 @@ App::delete('/v1/functions/:functionId') ->send(); } - Console::info("Deleting $results count"); + Console::info("Deleting " . count($results) . " deployments"); // Delete the containers of all deployments // TODO : @christy Delete all build containers as well. Not just the latest one, @@ -991,7 +991,7 @@ App::delete('/v1/functions/:functionId') $orchestration->remove('appwrite-function-' . $deployment['$id'], true); Console::info('Removed container for deployment ' . $deployment['$id']); - } catch (Throwable $th) { + } catch (\Throwable $th) { // Do nothing, we don't care that much if it fails Console::error($th->getMessage()); } finally { @@ -1041,48 +1041,56 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/runtime') App::delete('/v1/deployments/:deploymentId') ->desc('Delete a deployment') ->param('deploymentId', '', new UID(), 'Deployment unique ID.') + ->inject('projectId') ->inject('response') ->inject('dbForProject') - ->action(function (string $deploymentId, Response $response, Database $dbForProject) use ($orchestrationPool) { - try { - /** @var Orchestration $orchestration */ - $orchestration = $orchestrationPool->get(); + ->action(function (string $deploymentId, string $projectId, Response $response, Database $dbForProject) use ($orchestrationPool) { - // Get deployment document - $deployment = $dbForProject->getDocument('deployments', $deploymentId); + // Get deployment document + $deployment = $dbForProject->getDocument('deployments', $deploymentId); - // Check if deployment exists - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found', 404); - } + // Check if deployment exists + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found', 404); + } + global $register; + go(function () use ($projectId, $orchestrationPool, $register, $deployment) { try { + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + $cache = new Cache(new RedisCache($redis)); + $dbForProject = new Database(new MariaDB($db), $cache); + $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $dbForProject->setNamespace('_project_' . $projectId); + + $orchestration = $orchestrationPool->get(); // Remove any ongoing builds if ($deployment->getAttribute('buildId')) { $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - + if ($build->getAttribute('status') == 'building') { // Remove the build $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); Console::info('Removed build for deployment ' . $deployment['$id']); } } - + // Remove the container of the deployment $orchestration->remove('appwrite-function-' . $deployment['$id'], true); Console::info('Removed container for deployment ' . $deployment['$id']); - } catch (Exception $e) { + } catch (\Throwable $th) { // Do nothing, we don't care that much if it fails + Console::error($th->getMessage()); + } finally { + $orchestrationPool->put($orchestration); } - } catch (Exception $e) { - logError($e, "cleanupFunction"); - $orchestrationPool->put($orchestration); + + }); - return $response->json(['error' => $e->getMessage()]); - } - $orchestrationPool->put($orchestration); - - return $response->json(['success' => true]); + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->send(); }); App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') From a4e3eb07d85c9965f77821b42c0dddd25647c4cd Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 16:26:22 +0400 Subject: [PATCH 13/21] feat: put db and redis workers back in the pool --- app/executor.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/executor.php b/app/executor.php index 1e4ceee126..537767d2e6 100644 --- a/app/executor.php +++ b/app/executor.php @@ -972,12 +972,13 @@ App::delete('/v1/functions/:functionId') try { $db = $register->get('dbPool')->get(); $redis = $register->get('redisPool')->get(); + $orchestration = $orchestrationPool->get(); + $cache = new Cache(new RedisCache($redis)); $dbForProject = new Database(new MariaDB($db), $cache); $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $dbForProject->setNamespace('_project_' . $projectId); - $orchestration = $orchestrationPool->get(); // Remove any ongoing builds if ($deployment->getAttribute('buildId')) { $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); @@ -996,6 +997,8 @@ App::delete('/v1/functions/:functionId') Console::error($th->getMessage()); } finally { $orchestrationPool->put($orchestration); + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); } }); } @@ -1059,12 +1062,13 @@ App::delete('/v1/deployments/:deploymentId') try { $db = $register->get('dbPool')->get(); $redis = $register->get('redisPool')->get(); + $orchestration = $orchestrationPool->get(); + $cache = new Cache(new RedisCache($redis)); $dbForProject = new Database(new MariaDB($db), $cache); $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $dbForProject->setNamespace('_project_' . $projectId); - $orchestration = $orchestrationPool->get(); // Remove any ongoing builds if ($deployment->getAttribute('buildId')) { $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); @@ -1084,6 +1088,8 @@ App::delete('/v1/deployments/:deploymentId') Console::error($th->getMessage()); } finally { $orchestrationPool->put($orchestration); + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); } }); From 8d59a058a9863d2fa5dfefe8680128e5c80fb110 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 28 Jan 2022 16:26:58 +0400 Subject: [PATCH 14/21] feat: put db and redis workers back in the pool --- app/executor.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/executor.php b/app/executor.php index 537767d2e6..c16d8d4844 100644 --- a/app/executor.php +++ b/app/executor.php @@ -993,7 +993,6 @@ App::delete('/v1/functions/:functionId') $orchestration->remove('appwrite-function-' . $deployment['$id'], true); Console::info('Removed container for deployment ' . $deployment['$id']); } catch (\Throwable $th) { - // Do nothing, we don't care that much if it fails Console::error($th->getMessage()); } finally { $orchestrationPool->put($orchestration); @@ -1084,7 +1083,6 @@ App::delete('/v1/deployments/:deploymentId') $orchestration->remove('appwrite-function-' . $deployment['$id'], true); Console::info('Removed container for deployment ' . $deployment['$id']); } catch (\Throwable $th) { - // Do nothing, we don't care that much if it fails Console::error($th->getMessage()); } finally { $orchestrationPool->put($orchestration); From 220a1664349b2037dd55fcf54d461b6dbf299f23 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 29 Jan 2022 04:00:25 +0400 Subject: [PATCH 15/21] feat: fix tests --- app/executor.php | 32 ++++++++----------- app/workers/deletes.php | 2 +- .../Functions/FunctionsCustomClientTest.php | 29 ++++++++++++++--- .../Functions/FunctionsCustomServerTest.php | 20 +++++++++++- 4 files changed, 57 insertions(+), 26 deletions(-) diff --git a/app/executor.php b/app/executor.php index c16d8d4844..2e8b4185a1 100644 --- a/app/executor.php +++ b/app/executor.php @@ -962,8 +962,6 @@ App::delete('/v1/functions/:functionId') ->send(); } - Console::info("Deleting " . count($results) . " deployments"); - // Delete the containers of all deployments // TODO : @christy Delete all build containers as well. Not just the latest one, global $register; @@ -1049,15 +1047,10 @@ App::delete('/v1/deployments/:deploymentId') ->action(function (string $deploymentId, string $projectId, Response $response, Database $dbForProject) use ($orchestrationPool) { // Get deployment document - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - // Check if deployment exists - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found', 404); - } + // $deployment = $dbForProject->getDocument('deployments', $deploymentId); global $register; - go(function () use ($projectId, $orchestrationPool, $register, $deployment) { + go(function () use ($projectId, $orchestrationPool, $register, $deploymentId) { try { $db = $register->get('dbPool')->get(); $redis = $register->get('redisPool')->get(); @@ -1069,19 +1062,20 @@ App::delete('/v1/deployments/:deploymentId') $dbForProject->setNamespace('_project_' . $projectId); // Remove any ongoing builds - if ($deployment->getAttribute('buildId')) { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); + // TODO: Delete builds + // if ($deployment->getAttribute('buildId')) { + // $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - if ($build->getAttribute('status') == 'building') { - // Remove the build - $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); - Console::info('Removed build for deployment ' . $deployment['$id']); - } - } + // if ($build->getAttribute('status') == 'building') { + // // Remove the build + // $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); + // Console::info('Removed build for deployment ' . $deployment['$id']); + // } + // } // Remove the container of the deployment - $orchestration->remove('appwrite-function-' . $deployment['$id'], true); - Console::info('Removed container for deployment ' . $deployment['$id']); + $orchestration->remove('appwrite-function-' . $deploymentId , true); + Console::info('Removed container for deployment ' . $deploymentId); } catch (\Throwable $th) { Console::error($th->getMessage()); } finally { diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 8ae2bbe41c..96f9ebf327 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -419,7 +419,7 @@ class DeletesV1 extends Worker } /** - * Delete Deployments + * Delete deployment files */ $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index a175bb48a9..403fd7e450 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -117,14 +117,14 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(201, $execution['headers']['status-code']); - $execution = $this->client->call(Client::METHOD_POST, '/functions/'.$function['body']['$id'].'/executions', array_merge([ + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/'.$function['body']['$id'], [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], - ]), [ - 'async' => true, - ]); + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); - $this->assertEquals(401, $execution['headers']['status-code']); + $this->assertEquals(204, $response['headers']['status-code']); return []; } @@ -300,6 +300,16 @@ class FunctionsCustomClientTest extends Scope 'cursor' => $base['body']['executions'][1]['$id'], 'cursorDirection' => Database::CURSOR_BEFORE ]); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/'. $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } public function testSynchronousExecution():array @@ -382,6 +392,15 @@ class FunctionsCustomClientTest extends Scope $this->assertNotEmpty($output['APPWRITE_FUNCTION_JWT']); $this->assertEquals($projectId, $output['APPWRITE_FUNCTION_PROJECT_ID']); + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/'. $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + return []; } } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index a9e9096fe6..d740e39426 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -660,7 +660,7 @@ class FunctionsCustomServerTest extends Scope /** * Test for SUCCESS */ - $function = $this->client->call(Client::METHOD_DELETE, '/functions/'.$data['functionId'], array_merge([ + $function = $this->client->call(Client::METHOD_DELETE, '/functions/'. $data['functionId'], array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); @@ -760,6 +760,15 @@ class FunctionsCustomServerTest extends Scope $this->assertLessThan(3, $executions['body']['executions'][0]['time']); $this->assertEquals($executions['body']['executions'][0]['stdout'], ''); $this->assertEquals($executions['body']['executions'][0]['stderr'], 'Execution timed out.'); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/'. $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); } /** @@ -861,6 +870,15 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals($executions['body']['executions'][0]['$id'], $executionId); $this->assertEquals($executions['body']['executions'][0]['trigger'], 'http'); $this->assertStringContainsString('foobar', $executions['body']['executions'][0]['stdout']); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/'. $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); } public function testGetRuntimes() From f3dfec8e7534f2647677bacfeebb0d9ae737984b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 29 Jan 2022 04:52:06 +0400 Subject: [PATCH 16/21] feat: fix tests --- app/executor.php | 68 ++++++++----------- app/workers/deletes.php | 28 +++++--- .../Functions/FunctionsCustomServerTest.php | 10 +-- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/app/executor.php b/app/executor.php index 2e8b4185a1..d8283447f0 100644 --- a/app/executor.php +++ b/app/executor.php @@ -963,33 +963,32 @@ App::delete('/v1/functions/:functionId') } // Delete the containers of all deployments - // TODO : @christy Delete all build containers as well. Not just the latest one, global $register; foreach ($results as $deployment) { go(function () use ($orchestrationPool, $deployment, $register, $projectId) { try { + $orchestration = $orchestrationPool->get(); + // Remove the container of the deployment + $orchestration->remove('appwrite-function-' . $deployment['$id'], true); + Console::success('Removed container for deployment: ' . $deployment['$id']); + $db = $register->get('dbPool')->get(); $redis = $register->get('redisPool')->get(); - $orchestration = $orchestrationPool->get(); - $cache = new Cache(new RedisCache($redis)); $dbForProject = new Database(new MariaDB($db), $cache); $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $dbForProject->setNamespace('_project_' . $projectId); - // Remove any ongoing builds - if ($deployment->getAttribute('buildId')) { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - - if ($build->getAttribute('status') === 'building') { - // Remove the build - $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); - Console::info('Removed build for deployment ' . $deployment['$id']); - } + $builds = $dbForProject->find('builds', [ + new Query('deploymentId', Query::TYPE_EQUAL, [$deployment['$id']]), + new Query('status', Query::TYPE_EQUAL, ['building']) + ], 999); + + // Remove all the build containers + foreach ($builds as $build) { + $orchestration->remove('build-stage-' . $build['$id'], true); + Console::success("Removed build contanier: $build for deployment: " . $deployment['$id']); } - - $orchestration->remove('appwrite-function-' . $deployment['$id'], true); - Console::info('Removed container for deployment ' . $deployment['$id']); } catch (\Throwable $th) { Console::error($th->getMessage()); } finally { @@ -1043,39 +1042,33 @@ App::delete('/v1/deployments/:deploymentId') ->param('deploymentId', '', new UID(), 'Deployment unique ID.') ->inject('projectId') ->inject('response') - ->inject('dbForProject') - ->action(function (string $deploymentId, string $projectId, Response $response, Database $dbForProject) use ($orchestrationPool) { - - // Get deployment document - // $deployment = $dbForProject->getDocument('deployments', $deploymentId); + ->action(function (string $deploymentId, string $projectId, Response $response) use ($orchestrationPool) { global $register; go(function () use ($projectId, $orchestrationPool, $register, $deploymentId) { try { + $orchestration = $orchestrationPool->get(); + // Remove the container of the deployment + $orchestration->remove('appwrite-function-' . $deploymentId , true); + Console::success('Removed container for deployment: ' . $deploymentId); + $db = $register->get('dbPool')->get(); $redis = $register->get('redisPool')->get(); - $orchestration = $orchestrationPool->get(); - $cache = new Cache(new RedisCache($redis)); $dbForProject = new Database(new MariaDB($db), $cache); $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $dbForProject->setNamespace('_project_' . $projectId); - // Remove any ongoing builds - // TODO: Delete builds - // if ($deployment->getAttribute('buildId')) { - // $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - - // if ($build->getAttribute('status') == 'building') { - // // Remove the build - // $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); - // Console::info('Removed build for deployment ' . $deployment['$id']); - // } - // } - - // Remove the container of the deployment - $orchestration->remove('appwrite-function-' . $deploymentId , true); - Console::info('Removed container for deployment ' . $deploymentId); + $builds = $dbForProject->find('builds', [ + new Query('deploymentId', Query::TYPE_EQUAL, [$deploymentId]), + new Query('status', Query::TYPE_EQUAL, ['building']) + ], 999); + + // Remove all the build containers + foreach ($builds as $build) { + $orchestration->remove('build-stage-' . $build['$id'], true); + Console::success("Removed build container: $build for deployment: " . $deploymentId); + } } catch (\Throwable $th) { Console::error($th->getMessage()); } finally { @@ -1083,7 +1076,6 @@ App::delete('/v1/deployments/:deploymentId') $register->get('dbPool')->put($db); $register->get('redisPool')->put($redis); } - }); $response diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 96f9ebf327..e74f09f803 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -361,16 +361,18 @@ class DeletesV1 extends Worker /** * Delete builds */ - $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); - $this->deleteByGroup('builds', [ - new Query('deploymentId', Query::TYPE_EQUAL, $deploymentIds) - ], $dbForProject, function (Document $document) use ($storageBuilds) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); - } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); - } - }); + if (!empty($deploymentIds)) { + $storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId); + $this->deleteByGroup('builds', [ + new Query('deploymentId', Query::TYPE_EQUAL, $deploymentIds) + ], $dbForProject, function (Document $document) use ($storageBuilds) { + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + } else { + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + } + }); + } // Delete Executions $this->deleteByGroup('executions', [ @@ -525,7 +527,11 @@ class DeletesV1 extends Worker while ($sum === $limit) { $chunk++; - $results = Authorization::skip(fn() => $database->find($collection, $queries, $limit, 0)); + Authorization::disable(); + + $results = $database->find($collection, $queries, $limit, 0); + + Authorization::reset(); $sum = count($results); diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index d740e39426..c5784e5739 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -668,12 +668,12 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $function['headers']['status-code']); $this->assertEmpty($function['body']); - $function = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'], array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + // $function = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'], array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'], + // ], $this->getHeaders())); - $this->assertEquals(404, $function['headers']['status-code']); + // $this->assertEquals(404, $function['headers']['status-code']); /** * Test for FAILURE From e816edc04716513d67e85cc8f0422014b45c10a8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 29 Jan 2022 04:54:47 +0400 Subject: [PATCH 17/21] feat: fix tests --- .../Services/Functions/FunctionsCustomServerTest.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index c5784e5739..d740e39426 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -668,12 +668,12 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $function['headers']['status-code']); $this->assertEmpty($function['body']); - // $function = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'], array_merge([ - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'], - // ], $this->getHeaders())); + $function = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); - // $this->assertEquals(404, $function['headers']['status-code']); + $this->assertEquals(404, $function['headers']['status-code']); /** * Test for FAILURE From 0de8ff6d4168da5d5ee9aba50e142bb649721466 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 31 Jan 2022 18:43:11 +0400 Subject: [PATCH 18/21] feat: merge conflicts --- app/executor.php | 16 ++++++++-------- app/workers/deletes.php | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/executor.php b/app/executor.php index 39ba408622..46582ab3ae 100644 --- a/app/executor.php +++ b/app/executor.php @@ -962,7 +962,7 @@ App::delete('/v1/functions/:functionId') ->action( function (string $functionId, string $projectId, Response $response, Database $dbForProject) use ($orchestrationPool) { - $results = $dbForProject->find('deployments', [new Query('functionId', Query::TYPE_EQUAL, [$functionId])], 999); + $results = $dbForProject->find('deployments', [new Query('resourceId', Query::TYPE_EQUAL, [$functionId])], 999); // If amount is 0 then we simply return true if (count($results) === 0) { @@ -975,19 +975,19 @@ App::delete('/v1/functions/:functionId') global $register; foreach ($results as $deployment) { go(function () use ($orchestrationPool, $deployment, $register, $projectId) { + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + $cache = new Cache(new RedisCache($redis)); + $dbForProject = new Database(new MariaDB($db), $cache); + $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $dbForProject->setNamespace('_project_' . $projectId); + try { $orchestration = $orchestrationPool->get(); // Remove the container of the deployment $orchestration->remove('appwrite-function-' . $deployment['$id'], true); Console::success('Removed container for deployment: ' . $deployment['$id']); - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - $cache = new Cache(new RedisCache($redis)); - $dbForProject = new Database(new MariaDB($db), $cache); - $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $dbForProject->setNamespace('_project_' . $projectId); - $builds = $dbForProject->find('builds', [ new Query('deploymentId', Query::TYPE_EQUAL, [$deployment['$id']]), new Query('status', Query::TYPE_EQUAL, ['building']) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index e74f09f803..e29defcad5 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -348,7 +348,7 @@ class DeletesV1 extends Worker $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); $deploymentIds = []; $this->deleteByGroup('deployments', [ - new Query('functionId', Query::TYPE_EQUAL, [$document->getId()]) + new Query('resourceId', Query::TYPE_EQUAL, [$document->getId()]) ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { $deploymentIds[] = $document->getId(); if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { From 7d82c1006aa5abe0d3ce13bbf2ffc6a16ba263f4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 1 Feb 2022 03:44:55 +0400 Subject: [PATCH 19/21] feat: fix tests --- app/controllers/api/functions.php | 3 ++- app/executor.php | 18 +++++---------- app/workers/builds.php | 22 ++++++------------- app/workers/deletes.php | 4 ++-- .../Functions/FunctionsCustomClientTest.php | 6 +++-- .../Functions/FunctionsCustomServerTest.php | 9 ++++---- tests/unit/Docker/ComposeTest.php | 2 +- 7 files changed, 26 insertions(+), 38 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index e34c4cf01f..6290abd81e 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -368,6 +368,7 @@ App::patch('/v1/functions/:functionId/deployment') $function = $dbForProject->getDocument('functions', $functionId); $deployment = $dbForProject->getDocument('deployments', $deployment); + var_dump($deployment); $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($function->isEmpty()) { @@ -699,7 +700,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') throw new Exception('Deployment not found', 404); } - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found', 404); } diff --git a/app/executor.php b/app/executor.php index 46582ab3ae..982a9b8cee 100644 --- a/app/executor.php +++ b/app/executor.php @@ -408,7 +408,7 @@ function execute(string $trigger, string $projectId, string $executionId, string $buildId = $database->getId(); $database->createDocument('builds', new Document([ '$id' => $buildId, - '$read' => ($userId !== '') ? ['user:' . $userId] : [], + '$read' => [], '$write' => [], 'startTime' => time(), 'deploymentId' => $deployment->getId(), @@ -685,9 +685,7 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) // Update deployment Status $build->setAttribute('status', 'building'); - $deployment->setAttribute('status', 'building'); $database->updateDocument('builds', $buildId, $build); - $database->updateDocument('deployments', $deploymentId, $deployment); // Check if runtime is active $runtime = $runtimes[$build->getAttribute('runtime', '')] ?? null; @@ -903,14 +901,11 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) ->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096))) ->setAttribute('startTime', $buildStart) - ->setAttribute('endTime', \microtime(true)) - ->setAttribute('duration', \microtime(true) - $buildStart); + ->setAttribute('endTime', \time()) + ->setAttribute('duration', \time() - $buildStart); $build = $database->updateDocument('builds', $buildId, $build); - $deployment->setAttribute('status', 'failed'); - $database->updateDocument('deployments', $deploymentId, $deployment); - // also remove the container if it exists if (isset($id)) { $orchestration->remove($id, true); @@ -971,6 +966,7 @@ App::delete('/v1/functions/:functionId') ->send(); } + Console::info('Deleting function: ' . $functionId); // Delete the containers of all deployments global $register; foreach ($results as $deployment) { @@ -1052,7 +1048,7 @@ App::delete('/v1/deployments/:deploymentId') ->inject('projectId') ->inject('response') ->action(function (string $deploymentId, string $projectId, Response $response) use ($orchestrationPool) { - + Console::info('Deleting deployment: ' . $deploymentId); global $register; go(function () use ($projectId, $orchestrationPool, $register, $deploymentId) { try { @@ -1154,12 +1150,10 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') // Deploy Runtime Server try { - Console::info("[ INFO ] Creating runtime server"); + Console::info("Creating runtime server"); createRuntimeServer($functionId, $projectId, $deploymentId, $dbForProject); } catch (\Throwable $th) { Console::error($th->getMessage()); - $deployment->setAttribute('status', 'failed'); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); throw $th; } }); diff --git a/app/workers/builds.php b/app/workers/builds.php index 25792c8de0..acb8ed6633 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -37,7 +37,7 @@ class BuildsV1 extends Worker case BUILD_TYPE_DEPLOYMENT: $functionId = $this->args['functionId'] ?? ''; $deploymentId = $this->args['deploymentId'] ?? ''; - Console::info("[ INFO ] Creating build for deployment: $deploymentId"); + Console::info("Creating build for deployment: $deploymentId"); $this->buildDeployment($projectId, $functionId, $deploymentId); break; @@ -45,7 +45,7 @@ class BuildsV1 extends Worker $buildId = $this->args['buildId'] ?? ''; $functionId = $this->args['functionId'] ?? ''; $deploymentId = $this->args['deploymentId'] ?? ''; - Console::info("[ INFO ] Retrying build for id: $buildId"); + Console::info("Retrying build for id: $buildId"); $this->createBuild($projectId, $functionId, $deploymentId, $buildId); break; @@ -113,29 +113,21 @@ class BuildsV1 extends Worker if (empty($buildId)) { try { $buildId = $dbForProject->getId(); - // TODO : There is no way to associate a build with a deployment. So we need to add a deploymentId attribute to the build document $dbForProject->createDocument('builds', new Document([ '$id' => $buildId, '$read' => [], '$write' => [], + 'startTime' => time(), 'deploymentId' => $deploymentId, - 'dateCreated' => time(), 'status' => 'processing', - 'runtime' => $function->getAttribute('runtime'), 'outputPath' => '', + 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), 'sourceType' => Storage::DEVICE_LOCAL, 'stdout' => '', 'stderr' => '', - 'time' => 0, - 'vars' => [ - 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint'), - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], - 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, - ] + 'endTime' => 0, + 'duration' => 0 ])); } catch (\Throwable $th) { $deployment->setAttribute('buildId', ''); @@ -155,7 +147,7 @@ class BuildsV1 extends Worker throw $th; } - Console::success("[ SUCCESS ] Build id: $buildId started"); + Console::success("Build id: $buildId started"); } public function shutdown(): void {} diff --git a/app/workers/deletes.php b/app/workers/deletes.php index e29defcad5..7e46325cc3 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -352,7 +352,7 @@ class DeletesV1 extends Worker ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { $deploymentIds[] = $document->getId(); if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { - Console::success('Delete deployment files: ' . $document->getAttribute('path', '')); + Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); } else { Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); } @@ -425,7 +425,7 @@ class DeletesV1 extends Worker */ $storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { - Console::success('Delete deployment files: ' . $document->getAttribute('path', '')); + Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); } else { Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 403fd7e450..3255caf2a9 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -87,7 +87,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(201, $deployment['headers']['status-code']); // Wait for deployment to be built. - sleep(5); + sleep(10); $function = $this->client->call(Client::METHOD_PATCH, '/functions/'.$function['body']['$id'].'/deployment', [ 'content-type' => 'application/json', @@ -97,6 +97,8 @@ class FunctionsCustomClientTest extends Scope 'deployment' => $deploymentId, ]); + // var_dump($function); + $this->assertEquals(200, $function['headers']['status-code']); $execution = $this->client->call(Client::METHOD_POST, '/functions/'.$function['body']['$id'].'/executions', [ @@ -356,7 +358,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(201, $deployment['headers']['status-code']); // Wait for deployment to be built. - sleep(5); + sleep(10); $function = $this->client->call(Client::METHOD_PATCH, '/functions/'.$functionId.'/deployment', [ 'content-type' => 'application/json', diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index f7a0275e9f..9bb2a296e9 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -291,10 +291,9 @@ class FunctionsCustomServerTest extends Scope $this->assertNotEmpty($deployment['body']['$id']); $this->assertIsInt($deployment['body']['dateCreated']); $this->assertEquals('index.php', $deployment['body']['entrypoint']); - // $this->assertGreaterThan(10000, $deployment['body']['size']); // Wait for deployment to build. - sleep(5); + sleep(15); /** * Test for FAILURE @@ -450,7 +449,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals('', $execution['body']['stderr']); $this->assertEquals(0, $execution['body']['time']); - sleep(15); + sleep(5); $execution = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/executions/'.$executionId, array_merge([ 'content-type' => 'application/json', @@ -686,7 +685,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(201, $deployment['headers']['status-code']); // Allow build step to run - sleep(5); + sleep(10); $deployment = $this->client->call(Client::METHOD_PATCH, '/functions/'.$functionId.'/deployment', array_merge([ 'content-type' => 'application/json', @@ -778,7 +777,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(201, $deployment['headers']['status-code']); // Allow build step to run - sleep(5); + sleep(10); $deployment = $this->client->call(Client::METHOD_PATCH, '/functions/'.$functionId.'/deployment', array_merge([ 'content-type' => 'application/json', diff --git a/tests/unit/Docker/ComposeTest.php b/tests/unit/Docker/ComposeTest.php index 79fbdd9f95..c484fc17e8 100644 --- a/tests/unit/Docker/ComposeTest.php +++ b/tests/unit/Docker/ComposeTest.php @@ -36,7 +36,7 @@ class ComposeTest extends TestCase public function testServices() { - $this->assertCount(16, $this->object->getServices()); + $this->assertCount(17, $this->object->getServices()); $this->assertEquals('appwrite-telegraf', $this->object->getService('telegraf')->getContainerName()); $this->assertEquals('appwrite', $this->object->getService('appwrite')->getContainerName()); $this->assertEquals('', $this->object->getService('appwrite')->getImageVersion()); From 5e1d491cdd624694b607b45ea0103ca4167c9b0c Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 8 Feb 2022 10:46:55 +0000 Subject: [PATCH 20/21] Fix build entrypoint --- app/executor.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/executor.php b/app/executor.php index 982a9b8cee..34a4d93455 100644 --- a/app/executor.php +++ b/app/executor.php @@ -746,6 +746,7 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) } $vars = $resource->getAttribute('vars', []); + $vars['ENTRYPOINT_NAME'] = $resource->getAttribute('entrypoint', ''); $orchestration ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) From d0abbd9fc30f9bee69c34b01c237900ce2abf9b7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 13 Feb 2022 14:16:15 +0400 Subject: [PATCH 21/21] feat: disable authorization in deletes worker --- app/controllers/api/functions.php | 1 - app/workers/deletes.php | 14 ++++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 6290abd81e..fdb075daee 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -368,7 +368,6 @@ App::patch('/v1/functions/:functionId/deployment') $function = $dbForProject->getDocument('functions', $functionId); $deployment = $dbForProject->getDocument('deployments', $deployment); - var_dump($deployment); $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($function->isEmpty()) { diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 7e46325cc3..0449ed0d91 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -14,6 +14,8 @@ use Utopia\Audit\Audit; require_once __DIR__ . '/../init.php'; +Authorization::disable(); + Console::title('Deletes V1 Worker'); Console::success(APP_NAME . ' deletes worker v1 has started' . "\n"); @@ -206,7 +208,7 @@ class DeletesV1 extends Worker ], $this->getProjectDB($projectId)); $user->setAttribute('sessions', []); - $updated = Authorization::skip(fn() => $this->getProjectDB($projectId)->updateDocument('users', $userId, $user)); + $updated = $this->getProjectDB($projectId)->updateDocument('users', $userId, $user); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ @@ -456,8 +458,6 @@ class DeletesV1 extends Worker */ protected function deleteById(Document $document, Database $database, callable $callback = null): bool { - Authorization::disable(); - if ($database->deleteDocument($document->getCollection(), $document->getId())) { Console::success('Deleted document "' . $document->getId() . '" successfully'); @@ -470,8 +470,6 @@ class DeletesV1 extends Worker Console::error('Failed to delete document: ' . $document->getId()); return false; } - - Authorization::reset(); } /** @@ -488,7 +486,7 @@ class DeletesV1 extends Worker $executionStart = \microtime(true); while ($sum === $limit) { - $projects = Authorization::skip(fn() => $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit))); + $projects = $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit)); $chunk++; @@ -527,12 +525,8 @@ class DeletesV1 extends Worker while ($sum === $limit) { $chunk++; - Authorization::disable(); - $results = $database->find($collection, $queries, $limit, 0); - Authorization::reset(); - $sum = count($results); Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');