From d1ab8c4df8909e665155c3c601826eee624f0bc2 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:04:08 +0530 Subject: [PATCH 1/3] Move createDocument from api to worker --- app/controllers/general.php | 28 +++++++++++++-------- composer.lock | 26 +++++++++---------- src/Appwrite/Platform/Workers/Functions.php | 5 ++++ 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 04554a940e..f31d0522d8 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -6,6 +6,7 @@ use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Event; +use Appwrite\Event\Func; use Appwrite\Event\Usage; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Network\Validator\Origin; @@ -44,7 +45,7 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) +function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); @@ -366,7 +367,10 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->trigger() ; - $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); + $queueForFunctions + ->setType('delayed_execution_write') + ->setExecution($execution) + ->trigger(); } $execution->setAttribute('logs', ''); @@ -450,7 +454,8 @@ App::init() ->inject('queueForUsage') ->inject('queueForEvents') ->inject('queueForCertificates') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates) { + ->inject('queueForFunctions') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions) { /* * Appwrite Router */ @@ -458,7 +463,7 @@ App::init() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb)) { return; } } @@ -666,8 +671,9 @@ App::options() ->inject('getProjectDB') ->inject('queueForEvents') ->inject('queueForUsage') + ->inject('queueForFunctions') ->inject('geodb') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { /* * Appwrite Router */ @@ -675,7 +681,7 @@ App::options() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb)) { return; } } @@ -957,8 +963,9 @@ App::get('/robots.txt') ->inject('getProjectDB') ->inject('queueForEvents') ->inject('queueForUsage') + ->inject('queueForFunctions') ->inject('geodb') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -966,7 +973,7 @@ App::get('/robots.txt') $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb); } }); @@ -982,8 +989,9 @@ App::get('/humans.txt') ->inject('getProjectDB') ->inject('queueForEvents') ->inject('queueForUsage') + ->inject('queueForFunctions') ->inject('geodb') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -991,7 +999,7 @@ App::get('/humans.txt') $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb); } }); diff --git a/composer.lock b/composer.lock index bba2391d2c..4db97e4506 100644 --- a/composer.lock +++ b/composer.lock @@ -2124,16 +2124,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.12.0", + "version": "0.12.1", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "6e466d3511981291843c6ebf9ce3f44fc75e37b0" + "reference": "b9dfafb5efc1d12cbee01d03dc98853ef026e35b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/6e466d3511981291843c6ebf9ce3f44fc75e37b0", - "reference": "6e466d3511981291843c6ebf9ce3f44fc75e37b0", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/b9dfafb5efc1d12cbee01d03dc98853ef026e35b", + "reference": "b9dfafb5efc1d12cbee01d03dc98853ef026e35b", "shasum": "" }, "require": { @@ -2169,9 +2169,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.12.0" + "source": "https://github.com/utopia-php/messaging/tree/0.12.1" }, - "time": "2024-05-30T14:58:25+00:00" + "time": "2024-10-09T08:17:07+00:00" }, { "name": "utopia-php/migration", @@ -3573,16 +3573,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.3.0", + "version": "v5.3.1", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "3abf7425cd284141dc5d8d14a9ee444de3345d1a" + "reference": "8eea230464783aa9671db8eea6f8c6ac5285794b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/3abf7425cd284141dc5d8d14a9ee444de3345d1a", - "reference": "3abf7425cd284141dc5d8d14a9ee444de3345d1a", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/8eea230464783aa9671db8eea6f8c6ac5285794b", + "reference": "8eea230464783aa9671db8eea6f8c6ac5285794b", "shasum": "" }, "require": { @@ -3625,9 +3625,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.3.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.3.1" }, - "time": "2024-09-29T13:56:26+00:00" + "time": "2024-10-08T18:51:32+00:00" }, { "name": "phar-io/manifest", @@ -7028,5 +7028,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.3.0" } diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index dfe7435426..5b917b5ac9 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -209,6 +209,11 @@ class Functions extends Action executionId: $execution->getId() ?? null ); break; + case 'delayed_execution_write': + $execution = new Document($payload['execution'] ?? []); + $execution->setAttribute('$collection', 'executions'); + $execution = $dbForProject->createDocument('executions', $execution); + return; } } From 8fea73d3bf2dd093a202a63e63b8a6bb87e539c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 11 Oct 2024 13:32:48 +0000 Subject: [PATCH 2/3] Fix db issues --- app/controllers/general.php | 1 + src/Appwrite/Platform/Workers/Functions.php | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index f31d0522d8..afc126838d 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -370,6 +370,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $queueForFunctions ->setType('delayed_execution_write') ->setExecution($execution) + ->setProject($project) ->trigger(); } diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 5b917b5ac9..aa88f5cfce 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -72,6 +72,14 @@ class Functions extends Action } $type = $payload['type'] ?? ''; + + // Short-term solution to offhand write operation from API contianer + if ($type === 'delayed_execution_write') { + $execution = new Document($payload['execution'] ?? []); + $execution = $dbForProject->createDocument('executions', $execution); + return; + } + $events = $payload['events'] ?? []; $data = $payload['body'] ?? ''; $eventData = $payload['payload'] ?? ''; @@ -209,11 +217,6 @@ class Functions extends Action executionId: $execution->getId() ?? null ); break; - case 'delayed_execution_write': - $execution = new Document($payload['execution'] ?? []); - $execution->setAttribute('$collection', 'executions'); - $execution = $dbForProject->createDocument('executions', $execution); - return; } } From 408dcbc263150b3ca2c9204904ebc8ca63876eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 14 Oct 2024 13:06:08 +0000 Subject: [PATCH 3/3] Add async exec doc test --- app/controllers/general.php | 2 +- src/Appwrite/Event/Func.php | 2 ++ src/Appwrite/Platform/Workers/Functions.php | 2 +- .../Services/Functions/FunctionsCustomServerTest.php | 11 +++++++++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 03776144e6..b2a07f06f6 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -369,7 +369,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ; $queueForFunctions - ->setType('delayed_execution_write') + ->setType(Func::TYPE_ASYNC_WRITE) ->setExecution($execution) ->setProject($project) ->trigger(); diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 0cbaf17b60..4dad5802f7 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -8,6 +8,8 @@ use Utopia\Queue\Connection; class Func extends Event { + public const TYPE_ASYNC_WRITE = 'async_write'; + protected string $jwt = ''; protected string $type = ''; protected string $body = ''; diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index aa88f5cfce..7e548f57be 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -74,7 +74,7 @@ class Functions extends Action $type = $payload['type'] ?? ''; // Short-term solution to offhand write operation from API contianer - if ($type === 'delayed_execution_write') { + if ($type === Func::TYPE_ASYNC_WRITE) { $execution = new Document($payload['execution'] ?? []); $execution = $dbForProject->createDocument('executions', $execution); return; diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 59a3041a3f..9b9f03a100 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1675,6 +1675,17 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); $this->assertEquals($cookie, $response['body']); + // Async execution document creation + $this->assertEventually(function () use ($functionId) { + $executions = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $executions['headers']['status-code']); + $this->assertEquals(1, count($executions['body']['executions'])); + }); + // Await Aggregation sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30));