From bc631d926eeec8ef1b716650168c782617f9674a Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 4 Nov 2022 07:12:08 +0200 Subject: [PATCH 01/65] ticker --- Dockerfile | 1 + app/cli.php | 29 +++++++++ app/config/collections.php | 90 ++++++++++++++++++++++++++++ app/controllers/api/functions.php | 57 +++++++++++++++++- app/tasks/schedule.php | 97 +++++++++++++++++++++++++++++++ bin/schedule-new | 3 + docker-compose.yml | 33 ++++++++++- 7 files changed, 307 insertions(+), 3 deletions(-) create mode 100644 app/tasks/schedule.php create mode 100644 bin/schedule-new diff --git a/Dockerfile b/Dockerfile index 410fb1c44c..cfa588c9cf 100755 --- a/Dockerfile +++ b/Dockerfile @@ -349,6 +349,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/executor && \ chmod +x /usr/local/bin/schedule && \ + chmod +x /usr/local/bin/schedule-new && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ diff --git a/app/cli.php b/app/cli.php index 3a62c80816..7e4755bc58 100644 --- a/app/cli.php +++ b/app/cli.php @@ -10,6 +10,7 @@ use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use InfluxDB\Database as InfluxDatabase; @@ -59,6 +60,33 @@ function getConsoleDB(): Database return $database; } +/** + * Get internal project database + * @param Document $project + * @return Database + */ +function getProjectDB(Document $project): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + function getCache(): Cache { global $register; @@ -92,6 +120,7 @@ include 'tasks/specs.php'; include 'tasks/ssl.php'; include 'tasks/vars.php'; include 'tasks/usage.php'; +include 'tasks/schedule.php'; $cli ->task('version') diff --git a/app/config/collections.php b/app/config/collections.php index 9423e2eaac..f46251ed77 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -751,6 +751,96 @@ $collections = [ ], ], + 'schedules' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('schedules'), + 'name' => 'schedules', + 'attributes' => [ + [ + '$id' => ID::custom('type'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('scheduleId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('projectId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('scheduleUpdatedAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('schedule'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('active'), + 'type' => Database::VAR_BOOLEAN, + 'signed' => true, + 'size' => 0, + 'format' => '', + 'filters' => [], + 'required' => false, + 'default' => null, + 'array' => false, + ], + ], + 'indexes' => [ + [ + '$id' => ID::custom('_key_type_scheduleUpdatedAt'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['type','scheduleUpdatedAt'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => ID::custom('_key_type_projectId_scheduleId'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['type', 'projectId', 'scheduleId'], + 'lengths' => [], + 'orders' => [], + ], + ], + ], + 'platforms' => [ '$collection' => ID::custom(Database::METADATA), '$id' => ID::custom('platforms'), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index f2745281fc..9540ab3d7e 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -448,7 +448,8 @@ App::put('/v1/functions/:functionId') ->inject('project') ->inject('user') ->inject('events') - ->action(function (string $functionId, string $name, array $execute, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $name, array $execute, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); @@ -474,6 +475,36 @@ App::put('/v1/functions/:functionId') ]))); if ($next) { + $schedule = Authorization::skip(function () use ($dbForConsole, $project, $function) { + return $dbForConsole->findOne('schedules', [ + Query::equal('type', ['function']), + Query::equal('projectId', [$project->getId()]), + Query::equal('scheduleId', [$function->getId()]), + ]); + }); + + // TODO constrain with unique index ?? + if (empty($schedule)) { + Authorization::skip( + fn() => $dbForConsole->createDocument('schedules', new Document([ + 'type' => 'function', + 'scheduleId' => $function->getId(), + 'projectId' => $project->getId(), + 'scheduleUpdatedAt' => $function['scheduleUpdatedAt'], + 'schedule' => $function['schedule'], + 'active' => true, + ])) + ); + } else { + $schedule + ->setAttribute('scheduleUpdatedAt', $function['scheduleUpdatedAt']) + ->setAttribute('schedule', $function['schedule']) + ; + Authorization::skip(function () use ($dbForConsole, $schedule, $function) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); + }); + } + // Async task reschedule $functionEvent = new Func(); $functionEvent @@ -560,7 +591,10 @@ App::delete('/v1/functions/:functionId') ->inject('dbForProject') ->inject('deletes') ->inject('events') - ->action(function (string $functionId, Response $response, Database $dbForProject, Delete $deletes, Event $events) { + ->inject('project') + ->inject('dbForConsole') + + ->action(function (string $functionId, Response $response, Database $dbForProject, Delete $deletes, Event $events, Document $project, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); @@ -572,6 +606,25 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } + $schedule = Authorization::skip(function () use ($dbForConsole, $project, $function) { + return $dbForConsole->findOne('schedules', [ + Query::equal('type', ['function']), + Query::equal('projectId', [$project->getId()]), + Query::equal('scheduleId', [$function->getId()]), + ]); + }); + + if (!empty($schedule)) { + $schedule + ->setAttribute('scheduleUpdatedAt', DateTime::now()) + ->setAttribute('active', false) + ; + + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); + }); + } + $deletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($function); diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php new file mode 100644 index 0000000000..f33ab184b3 --- /dev/null +++ b/app/tasks/schedule.php @@ -0,0 +1,97 @@ +task('schedule-new') +->desc('Function scheduler task') +->action(function () use ($register) { + Console::title('Scheduler V1'); + Console::success(APP_NAME . ' Scheduler v1 has started'); + + sleep(4); + + $dbForConsole = getConsoleDB(); + $count = 0; + $limit = 50; + $sum = $limit; + $functions = []; + while ($sum === $limit) { + $results = $dbForConsole->find('schedules', [ + Query::equal('type', ['function']), + Query::equal('active', [true]), + Query::limit($limit) + ]); + + $sum = count($results); + foreach ($results as $document) { + $functions[$document['scheduleId']] = $document; + $count++; + } + } + + $lastValidationTime = DateTime::format((new \DateTime())->sub(\DateInterval::createFromDateString(FUNCTION_VALIDATION_TIMER . ' seconds'))); + + Co\run( + function () use ($dbForConsole, &$functions, &$lastValidationTime) { + Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($dbForConsole, &$functions, &$lastValidationTime) { + $time = DateTime::now(); + Console::success("Validation proc run at : $time"); + var_dump($lastValidationTime); + $count = 0; + $limit = 50; + $sum = $limit; + $tmp = []; + while ($sum === $limit) { + var_dump($lastValidationTime); + $results = $dbForConsole->find('schedules', [ + Query::equal('type', ['function']), + Query::greaterThan('scheduleUpdatedAt', $lastValidationTime), + Query::limit($limit) + ]); + + $lastValidationTime = DateTime::now(); + + $sum = count($results); + foreach ($results as $document) { + $tmp['scheduleId'] = $document; + $count++; + } + } + + foreach ($tmp as $document) { + $org = strtotime($functions[$document['scheduleId']]['scheduleUpdatedAt']); + $new = strtotime($document['scheduleUpdatedAt']); + var_dump($document['scheduleId']); + var_dump($document['active']); + if ($document['active'] === false) { + Console::error("Removing : {$document['scheduleId']}"); + unset($functions[$document['scheduleId']]); + } elseif (!isset($functions[$document['scheduleId']]) || $new > $org) { + Console::error("Updating : {$document['scheduleId']}"); + $functions[$document['scheduleId']] = $document; + } + $count++; + } + }); + + Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, $functions) { + $time = DateTime::now(); + Console::success("Enqueue proc run at : $time"); + foreach ($functions as $function) { + Console::info("Enqueueing : {$function->getid()}"); + } + }); + } + ); +}); diff --git a/bin/schedule-new b/bin/schedule-new new file mode 100644 index 0000000000..6e6e54266b --- /dev/null +++ b/bin/schedule-new @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php schedule-new $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ea70cf331a..78ffe832aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,6 +108,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev + - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis @@ -684,6 +685,36 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + appwrite-schedule-new: + entrypoint: schedule-new + <<: *x-logging + container_name: appwrite-schedule-new + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + appwrite-schedule: entrypoint: schedule <<: *x-logging @@ -732,7 +763,7 @@ services: # - RELAY_FROM_HOSTS=192.168.0.0/16 ; *.yourdomain.com # - SMARTHOST_HOST=smtp # - SMARTHOST_PORT=587 - + redis: image: redis:7.0.4-alpine <<: *x-logging From ed505fba7c036769c2e6afe3188590aceb8a08b4 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 6 Nov 2022 15:10:26 +0200 Subject: [PATCH 02/65] queue --- .env | 3 +- app/config/collections.php | 23 ++++-- app/controllers/api/functions.php | 3 + app/tasks/schedule.php | 113 +++++++++++++++++++++--------- docker-compose.yml | 2 + 5 files changed, 103 insertions(+), 41 deletions(-) diff --git a/.env b/.env index 950fd44821..a532b525e6 100644 --- a/.env +++ b/.env @@ -86,6 +86,7 @@ _APP_USAGE_DATABASE_INTERVAL=15 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= +_APP_REGION=nyc1 DOCKERHUB_PULL_USERNAME= DOCKERHUB_PULL_PASSWORD= -DOCKERHUB_PULL_EMAIL= \ No newline at end of file +DOCKERHUB_PULL_EMAIL= diff --git a/app/config/collections.php b/app/config/collections.php index f46251ed77..17f1fa8c41 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -760,7 +760,7 @@ $collections = [ '$id' => ID::custom('type'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => Database::LENGTH_KEY, + 'size' => 100, 'signed' => true, 'required' => false, 'default' => null, @@ -804,7 +804,7 @@ $collections = [ '$id' => ID::custom('schedule'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 256, + 'size' => 100, 'signed' => true, 'required' => false, 'default' => null, @@ -822,19 +822,30 @@ $collections = [ 'default' => null, 'array' => false, ], + [ + '$id' => ID::custom('region'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 10, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ - '$id' => ID::custom('_key_type_scheduleUpdatedAt'), + '$id' => ID::custom('_key_region_type_scheduleUpdatedAt'), 'type' => Database::INDEX_KEY, - 'attributes' => ['type','scheduleUpdatedAt'], + 'attributes' => ['region', 'type','scheduleUpdatedAt'], 'lengths' => [], 'orders' => [], ], [ - '$id' => ID::custom('_key_type_projectId_scheduleId'), + '$id' => ID::custom('_key_region_type_projectId_scheduleId'), 'type' => Database::INDEX_KEY, - 'attributes' => ['type', 'projectId', 'scheduleId'], + 'attributes' => ['region', 'type', 'projectId', 'scheduleId'], 'lengths' => [], 'orders' => [], ], diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 9540ab3d7e..a7c5dd7763 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -477,6 +477,7 @@ App::put('/v1/functions/:functionId') if ($next) { $schedule = Authorization::skip(function () use ($dbForConsole, $project, $function) { return $dbForConsole->findOne('schedules', [ + Query::equal('region', [App::getEnv('_APP_REGION')]), // Todo replace with projects region Query::equal('type', ['function']), Query::equal('projectId', [$project->getId()]), Query::equal('scheduleId', [$function->getId()]), @@ -487,6 +488,7 @@ App::put('/v1/functions/:functionId') if (empty($schedule)) { Authorization::skip( fn() => $dbForConsole->createDocument('schedules', new Document([ + 'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region 'type' => 'function', 'scheduleId' => $function->getId(), 'projectId' => $project->getId(), @@ -608,6 +610,7 @@ App::delete('/v1/functions/:functionId') $schedule = Authorization::skip(function () use ($dbForConsole, $project, $function) { return $dbForConsole->findOne('schedules', [ + Query::equal('region', [App::getEnv('_APP_REGION')]), // Todo replace with projects region Query::equal('type', ['function']), Query::equal('projectId', [$project->getId()]), Query::equal('scheduleId', [$function->getId()]), diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index f33ab184b3..65b7916af1 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -3,14 +3,16 @@ global $cli; global $register; +use Cron\CronExpression; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\DateTime; use Utopia\Database\Query; use Swoole\Timer; - -const FUNCTION_VALIDATION_TIMER = 30; //seconds -const FUNCTION_ENQUEUE_TIMER = 10; //seconds +const FUNCTION_VALIDATION_TIMER = 180; //seconds +const FUNCTION_ENQUEUE_TIMER = 60; //seconds +const ENQUEUE_TIME_FRAME = 60 * 5; // 5 min $cli ->task('schedule-new') @@ -19,15 +21,33 @@ $cli Console::title('Scheduler V1'); Console::success(APP_NAME . ' Scheduler v1 has started'); - sleep(4); + $createQueue = function () use (&$functions, &$queue) { + /** + * Creating smaller functions list containing 5-min timeframe. + */ + $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); + foreach ($functions as $function) { + $cron = new CronExpression($function['schedule']); + $next = DateTime::format($cron->getNextRunDate()); + if ($next < $timeFrame) { + $queue[$next][$function['scheduleId']] = $function; + } + } + }; $dbForConsole = getConsoleDB(); $count = 0; $limit = 50; $sum = $limit; $functions = []; + $queue = []; + + /** + * Initial run fill $functions list + */ while ($sum === $limit) { $results = $dbForConsole->find('schedules', [ + Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('type', ['function']), Query::equal('active', [true]), Query::limit($limit) @@ -40,56 +60,81 @@ $cli } } - $lastValidationTime = DateTime::format((new \DateTime())->sub(\DateInterval::createFromDateString(FUNCTION_VALIDATION_TIMER . ' seconds'))); + $createQueue(); + $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_VALIDATION_TIMER); Co\run( - function () use ($dbForConsole, &$functions, &$lastValidationTime) { - Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($dbForConsole, &$functions, &$lastValidationTime) { + function () use ($createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); - Console::success("Validation proc run at : $time"); - var_dump($lastValidationTime); $count = 0; $limit = 50; $sum = $limit; - $tmp = []; + + Console::info("Update proc run at: $time last update was at $lastUpdate"); + /** + * Updating functions list from DB. + */ while ($sum === $limit) { - var_dump($lastValidationTime); $results = $dbForConsole->find('schedules', [ + Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('type', ['function']), - Query::greaterThan('scheduleUpdatedAt', $lastValidationTime), + Query::greaterThan('scheduleUpdatedAt', $lastUpdate), Query::limit($limit) ]); - - $lastValidationTime = DateTime::now(); - $sum = count($results); foreach ($results as $document) { - $tmp['scheduleId'] = $document; + $org = isset($functions[$document['scheduleId']]) ? strtotime($functions[$document['scheduleId']]['scheduleUpdatedAt']) : null; + $new = strtotime($document['scheduleUpdatedAt']); + if ($document['active'] === false) { + Console::error("Removing: {$document['scheduleId']}"); + unset($functions[$document['scheduleId']]); + } elseif ($new > $org) { + Console::error("Updating: {$document['scheduleId']}"); + $functions[$document['scheduleId']] = $document; + } $count++; } } - foreach ($tmp as $document) { - $org = strtotime($functions[$document['scheduleId']]['scheduleUpdatedAt']); - $new = strtotime($document['scheduleUpdatedAt']); - var_dump($document['scheduleId']); - var_dump($document['active']); - if ($document['active'] === false) { - Console::error("Removing : {$document['scheduleId']}"); - unset($functions[$document['scheduleId']]); - } elseif (!isset($functions[$document['scheduleId']]) || $new > $org) { - Console::error("Updating : {$document['scheduleId']}"); - $functions[$document['scheduleId']] = $document; - } - $count++; - } + $lastUpdate = DateTime::now(); + $createQueue(); }); - Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, $functions) { + Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { $time = DateTime::now(); - Console::success("Enqueue proc run at : $time"); - foreach ($functions as $function) { - Console::info("Enqueueing : {$function->getid()}"); + $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */ + $now = (new \DateTime())->format('Y-m-d H:i:00.000'); + + Console::info("Enqueue proc run at: $time"); + + /** + * Lopping time slots + */ + foreach ($queue as $slot => $schedule) { + if ($now === $slot) { + foreach ($schedule as $function) { + /** + * Enqueue function + */ + Console::warning("Enqueueing :{$function['scheduleId']}"); + $cron = new CronExpression($function['schedule']); + $next = DateTime::format($cron->getNextRunDate()); + /** + * If next schedule is in 5-min timeframe + * and it was not removed re-enqueue the function. + */ + if ( + $next < $timeFrame && + !empty($functions[$function['scheduleId']]) + ) { + Console::warning("re-enqueueing :{$function['scheduleId']}"); + $queue[$next][$function['scheduleId']] = $function; + } + unset($queue[$slot][$function['scheduleId']]); /** removing function from slot */ + } + unset($queue[$slot]); /** removing slot */ + } } }); } diff --git a/docker-compose.yml b/docker-compose.yml index 78ffe832aa..34a7d1c2b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -183,6 +183,7 @@ services: - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_SMS_PROVIDER - _APP_SMS_FROM + - _APP_REGION appwrite-realtime: entrypoint: realtime @@ -714,6 +715,7 @@ services: - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_CACHE - _APP_CONNECTIONS_QUEUE + - _APP_REGION appwrite-schedule: entrypoint: schedule From 0e99198fa8dcd5fc60c42ed2820e03051c70abdc Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 6 Nov 2022 16:40:08 +0200 Subject: [PATCH 03/65] queue --- app/tasks/schedule.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 65b7916af1..b18f570af7 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -13,6 +13,7 @@ use Swoole\Timer; const FUNCTION_VALIDATION_TIMER = 180; //seconds const FUNCTION_ENQUEUE_TIMER = 60; //seconds const ENQUEUE_TIME_FRAME = 60 * 5; // 5 min +sleep(4); // Todo prevent PDOException $cli ->task('schedule-new') @@ -35,6 +36,17 @@ $cli } }; + $removeFromQueue = function ($scheduleId) use (&$queue) { + foreach ($queue as $slot => $schedule) { + foreach ($schedule as $function) { + if ($scheduleId === $function['scheduleId']) { + unset($queue[$slot][$function['scheduleId']]); + } + } + } + }; + + $dbForConsole = getConsoleDB(); $count = 0; $limit = 50; @@ -64,8 +76,8 @@ $cli $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_VALIDATION_TIMER); Co\run( - function () use ($createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { - Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $count = 0; $limit = 50; @@ -93,6 +105,7 @@ $cli Console::error("Updating: {$document['scheduleId']}"); $functions[$document['scheduleId']] = $document; } + $removeFromQueue($document['scheduleId']); $count++; } } From ef7b53fbc7577bed8f01926f607278473369f8b1 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 6 Nov 2022 23:41:33 +0200 Subject: [PATCH 04/65] queue --- Dockerfile | 1 - app/config/collections.php | 81 ++++++------------ app/controllers/api/functions.php | 136 +++++++++++------------------- app/tasks/schedule.php | 52 +++++++----- bin/schedule | 9 +- bin/schedule-new | 3 - docker-compose.yml | 27 +----- 7 files changed, 114 insertions(+), 195 deletions(-) delete mode 100644 bin/schedule-new diff --git a/Dockerfile b/Dockerfile index cfa588c9cf..410fb1c44c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -349,7 +349,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/executor && \ chmod +x /usr/local/bin/schedule && \ - chmod +x /usr/local/bin/schedule-new && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ diff --git a/app/config/collections.php b/app/config/collections.php index 17f1fa8c41..345d802dc4 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -757,7 +757,7 @@ $collections = [ 'name' => 'schedules', 'attributes' => [ [ - '$id' => ID::custom('type'), + '$id' => ID::custom('resourceType'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 100, @@ -768,7 +768,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('scheduleId'), + '$id' => ID::custom('resourceId'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => Database::LENGTH_KEY, @@ -778,6 +778,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('resourceUpdatedAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], [ '$id' => ID::custom('projectId'), 'type' => Database::VAR_STRING, @@ -789,17 +800,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('scheduleUpdatedAt'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], [ '$id' => ID::custom('schedule'), 'type' => Database::VAR_STRING, @@ -836,16 +836,16 @@ $collections = [ ], 'indexes' => [ [ - '$id' => ID::custom('_key_region_type_scheduleUpdatedAt'), + '$id' => ID::custom('_key_region_resourceType_resourceUpdatedAt'), 'type' => Database::INDEX_KEY, - 'attributes' => ['region', 'type','scheduleUpdatedAt'], + 'attributes' => ['region', 'resourceType','resourceUpdatedAt'], 'lengths' => [], 'orders' => [], ], [ - '$id' => ID::custom('_key_region_type_projectId_scheduleId'), + '$id' => ID::custom('_key_region_resourceType_projectId_resourceId'), 'type' => Database::INDEX_KEY, - 'attributes' => ['region', 'type', 'projectId', 'scheduleId'], + 'attributes' => ['region', 'resourceType', 'projectId', 'resourceId'], 'lengths' => [], 'orders' => [], ], @@ -2249,6 +2249,17 @@ $collections = [ 'array' => true, 'filters' => [], ], + [ + '$id' => ID::custom('scheduleId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('schedule'), 'type' => Database::VAR_STRING, @@ -2271,28 +2282,6 @@ $collections = [ 'array' => false, 'filters' => ['datetime'], ], - [ - '$id' => ID::custom('schedulePrevious'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('scheduleNext'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], [ '$id' => ID::custom('timeout'), 'type' => Database::VAR_INTEGER, @@ -2359,20 +2348,6 @@ $collections = [ 'lengths' => [128], 'orders' => [Database::ORDER_ASC], ], - [ - '$id' => ID::custom('_key_scheduleNext'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['scheduleNext'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_schedulePrevious'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['schedulePrevious'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], [ '$id' => ID::custom('_key_timeout'), 'type' => Database::INDEX_KEY, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a7c5dd7763..0edfa26eff 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -36,7 +36,6 @@ use Utopia\Validator\Text; use Utopia\Validator\Range; use Utopia\Validator\WhiteList; use Utopia\Config\Config; -use Cron\CronExpression; use Executor\Executor; use Utopia\CLI\Console; use Utopia\Database\Validator\Roles; @@ -72,10 +71,8 @@ App::post('/v1/functions') ->inject('project') ->inject('user') ->inject('events') - ->action(function (string $functionId, string $name, array $execute, string $runtime, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance) { - - $cron = !empty($schedule) ? new CronExpression($schedule) : null; - $next = !empty($schedule) ? DateTime::format($cron->getNextRunDate()) : null; + ->inject('dbForConsole') + ->action(function (string $functionId, string $name, array $execute, string $runtime, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; $function = $dbForProject->createDocument('functions', new Document([ @@ -88,22 +85,24 @@ App::post('/v1/functions') 'events' => $events, 'schedule' => $schedule, 'scheduleUpdatedAt' => DateTime::now(), - 'schedulePrevious' => null, - 'scheduleNext' => $next, 'timeout' => $timeout, 'search' => implode(' ', [$functionId, $name, $runtime]) ])); - if ($next) { - // Async task reschedule - $functionEvent = new Func(); - $functionEvent - ->setFunction($function) - ->setType('schedule') - ->setUser($user) - ->setProject($project) - ->schedule(new \DateTime($next)); - } + $log = Authorization::skip( + fn() => $dbForConsole->createDocument('schedules', new Document([ + 'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region + 'resourceType' => 'function', + 'resourceId' => $function->getId(), + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $project->getId(), + 'schedule' => $function['schedule'], + 'active' => false, + ])) + ); + + $function->setAttribute('scheduleId', $log->getId()); + $dbForProject->updateDocument('functions', $function->getId(), $function); $eventsInstance->setParam('functionId', $function->getId()); @@ -457,66 +456,25 @@ App::put('/v1/functions/:functionId') throw new Exception(Exception::FUNCTION_NOT_FOUND); } - $cron = !empty($schedule) ? new CronExpression($schedule) : null; - $next = !empty($schedule) ? DateTime::format($cron->getNextRunDate()) : null; - $enabled ??= $function->getAttribute('enabled', true); + $log = $schedule + ->setAttribute('resourceUpdatedAt', $function['resourceUpdatedAt']) + ->setAttribute('schedule', $function['schedule']) + ->setAttribute('active', !empty($function->getAttribute('schedule') || !empty($schedule))); + $function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [ 'execute' => $execute, 'name' => $name, 'events' => $events, + 'scheduleId' => $log->getId(), 'schedule' => $schedule, 'scheduleUpdatedAt' => DateTime::now(), - 'scheduleNext' => $next, 'timeout' => $timeout, 'enabled' => $enabled, 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); - if ($next) { - $schedule = Authorization::skip(function () use ($dbForConsole, $project, $function) { - return $dbForConsole->findOne('schedules', [ - Query::equal('region', [App::getEnv('_APP_REGION')]), // Todo replace with projects region - Query::equal('type', ['function']), - Query::equal('projectId', [$project->getId()]), - Query::equal('scheduleId', [$function->getId()]), - ]); - }); - - // TODO constrain with unique index ?? - if (empty($schedule)) { - Authorization::skip( - fn() => $dbForConsole->createDocument('schedules', new Document([ - 'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region - 'type' => 'function', - 'scheduleId' => $function->getId(), - 'projectId' => $project->getId(), - 'scheduleUpdatedAt' => $function['scheduleUpdatedAt'], - 'schedule' => $function['schedule'], - 'active' => true, - ])) - ); - } else { - $schedule - ->setAttribute('scheduleUpdatedAt', $function['scheduleUpdatedAt']) - ->setAttribute('schedule', $function['schedule']) - ; - Authorization::skip(function () use ($dbForConsole, $schedule, $function) { - $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); - }); - } - - // Async task reschedule - $functionEvent = new Func(); - $functionEvent - ->setFunction($function) - ->setType('schedule') - ->setUser($user) - ->setProject($project) - ->schedule(new \DateTime($next)); - } - $eventsInstance->setParam('functionId', $function->getId()); $response->dynamic($function, Response::MODEL_FUNCTION); @@ -542,7 +500,8 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') ->inject('dbForProject') ->inject('project') ->inject('events') - ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $events) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $events, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -568,6 +527,14 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') 'deployment' => $deployment->getId() ]))); + $log = $dbForProject->getDocument('schedules', $function['resourceId']); + + $log->setAttribute('active', true); + + Authorization::skip(function () use ($dbForConsole, $log) { + $dbForConsole->updateDocument('schedules', $log->getId(), $log); + }); + $events ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); @@ -595,7 +562,6 @@ App::delete('/v1/functions/:functionId') ->inject('events') ->inject('project') ->inject('dbForConsole') - ->action(function (string $functionId, Response $response, Database $dbForProject, Delete $deletes, Event $events, Document $project, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); @@ -608,26 +574,17 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } - $schedule = Authorization::skip(function () use ($dbForConsole, $project, $function) { - return $dbForConsole->findOne('schedules', [ - Query::equal('region', [App::getEnv('_APP_REGION')]), // Todo replace with projects region - Query::equal('type', ['function']), - Query::equal('projectId', [$project->getId()]), - Query::equal('scheduleId', [$function->getId()]), - ]); + $log = $dbForProject->getDocument('schedules', $function['resourceId']); + + $log + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('active', false) + ; + + Authorization::skip(function () use ($dbForConsole, $log) { + $dbForConsole->updateDocument('schedules', $log->getId(), $log); }); - if (!empty($schedule)) { - $schedule - ->setAttribute('scheduleUpdatedAt', DateTime::now()) - ->setAttribute('active', false) - ; - - Authorization::skip(function () use ($dbForConsole, $schedule) { - $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); - }); - } - $deletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($function); @@ -664,7 +621,8 @@ App::post('/v1/functions/:functionId/deployments') ->inject('project') ->inject('deviceFunctions') ->inject('deviceLocal') - ->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); @@ -816,6 +774,14 @@ App::post('/v1/functions/:functionId/deployments') } } + $log = $dbForProject->getDocument('schedules', $function['resourceId']); + + $log->setAttribute('active', true); + + Authorization::skip(function () use ($dbForConsole, $log) { + $dbForConsole->updateDocument('schedules', $log->getId(), $log); + }); + $metadata = null; $events diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index b18f570af7..4664c8c10f 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -16,7 +16,7 @@ const ENQUEUE_TIME_FRAME = 60 * 5; // 5 min sleep(4); // Todo prevent PDOException $cli -->task('schedule-new') +->task('schedule') ->desc('Function scheduler task') ->action(function () use ($register) { Console::title('Scheduler V1'); @@ -31,7 +31,7 @@ $cli $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); if ($next < $timeFrame) { - $queue[$next][$function['scheduleId']] = $function; + $queue[$next][$function['resourceId']] = $function; } } }; @@ -39,8 +39,9 @@ $cli $removeFromQueue = function ($scheduleId) use (&$queue) { foreach ($queue as $slot => $schedule) { foreach ($schedule as $function) { - if ($scheduleId === $function['scheduleId']) { - unset($queue[$slot][$function['scheduleId']]); + if ($scheduleId === $function['resourceId']) { + Console::error("Unsetting :{$function['resourceId']} from queue slot $slot"); + unset($queue[$slot][$function['resourceId']]); } } } @@ -60,14 +61,14 @@ $cli while ($sum === $limit) { $results = $dbForConsole->find('schedules', [ Query::equal('region', [App::getEnv('_APP_REGION')]), - Query::equal('type', ['function']), + Query::equal('resourceType', ['function']), Query::equal('active', [true]), Query::limit($limit) ]); $sum = count($results); foreach ($results as $document) { - $functions[$document['scheduleId']] = $document; + $functions[$document['resourceId']] = $document; $count++; } } @@ -90,22 +91,22 @@ $cli while ($sum === $limit) { $results = $dbForConsole->find('schedules', [ Query::equal('region', [App::getEnv('_APP_REGION')]), - Query::equal('type', ['function']), - Query::greaterThan('scheduleUpdatedAt', $lastUpdate), + Query::equal('resourceType', ['function']), + Query::greaterThan('resourceUpdatedAt', $lastUpdate), Query::limit($limit) ]); $sum = count($results); foreach ($results as $document) { - $org = isset($functions[$document['scheduleId']]) ? strtotime($functions[$document['scheduleId']]['scheduleUpdatedAt']) : null; - $new = strtotime($document['scheduleUpdatedAt']); + $org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null; + $new = strtotime($document['resourceUpdatedAt']); if ($document['active'] === false) { - Console::error("Removing: {$document['scheduleId']}"); - unset($functions[$document['scheduleId']]); + Console::error("Removing: {$document['resourceId']}"); + unset($functions[$document['resourceId']]); } elseif ($new > $org) { - Console::error("Updating: {$document['scheduleId']}"); - $functions[$document['scheduleId']] = $document; + Console::error("Updating: {$document['resourceId']}"); + $functions[$document['resourceId']] = $document; } - $removeFromQueue($document['scheduleId']); + $removeFromQueue($document['resourceId']); $count++; } } @@ -120,31 +121,40 @@ $cli $now = (new \DateTime())->format('Y-m-d H:i:00.000'); Console::info("Enqueue proc run at: $time"); + // Debug + foreach ($queue as $slot => $schedule) { + Console::log("Slot: $slot"); + foreach ($schedule as $function) { + Console::log("{$function['resourceId']} {$function['schedule']}"); + } + } /** * Lopping time slots */ + foreach ($queue as $slot => $schedule) { if ($now === $slot) { foreach ($schedule as $function) { /** * Enqueue function */ - Console::warning("Enqueueing :{$function['scheduleId']}"); + Console::warning("Enqueueing :{$function['resourceId']}"); $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); /** * If next schedule is in 5-min timeframe - * and it was not removed re-enqueue the function. + * and it was not removed or changed, re-enqueue the function. */ if ( $next < $timeFrame && - !empty($functions[$function['scheduleId']]) + !empty($functions[$function['resourceId']] && + $function['schedule'] === $functions[$function['resourceId']]['schedule']) ) { - Console::warning("re-enqueueing :{$function['scheduleId']}"); - $queue[$next][$function['scheduleId']] = $function; + Console::warning("re-enqueueing :{$function['resourceId']}"); + $queue[$next][$function['resourceId']] = $function; } - unset($queue[$slot][$function['scheduleId']]); /** removing function from slot */ + unset($queue[$slot][$function['resourceId']]); /** removing function from slot */ } unset($queue[$slot]); /** removing slot */ } diff --git a/bin/schedule b/bin/schedule index dbc6d94d96..ddd1ea7f35 100644 --- a/bin/schedule +++ b/bin/schedule @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 REDIS_BACKEND=$REDIS_BACKEND RESQUE_PHP='/usr/src/code/vendor/autoload.php' php /usr/src/code/vendor/bin/resque-scheduler +php /usr/src/code/app/cli.php schedule $@ \ No newline at end of file diff --git a/bin/schedule-new b/bin/schedule-new deleted file mode 100644 index 6e6e54266b..0000000000 --- a/bin/schedule-new +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php schedule-new $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 34a7d1c2b7..0dac1dc094 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -686,17 +686,16 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - appwrite-schedule-new: - entrypoint: schedule-new + appwrite-schedule: + entrypoint: schedule <<: *x-logging - container_name: appwrite-schedule-new + container_name: appwrite-schedule image: appwrite-dev networks: - appwrite volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis @@ -717,26 +716,6 @@ services: - _APP_CONNECTIONS_QUEUE - _APP_REGION - appwrite-schedule: - entrypoint: schedule - <<: *x-logging - container_name: appwrite-schedule - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - environment: - - _APP_ENV - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_CONNECTIONS_QUEUE - mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p container_name: appwrite-mariadb From 6f9dcae8588d97882aa57241566f58b85678ffd3 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 7 Nov 2022 12:37:07 +0200 Subject: [PATCH 05/65] queue --- app/controllers/api/functions.php | 41 ++++++++++++++++++++++--------- app/tasks/schedule.php | 34 ++++++++++++++++++------- docker-compose.yml | 2 +- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0edfa26eff..c73fbb686c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -458,16 +458,10 @@ App::put('/v1/functions/:functionId') $enabled ??= $function->getAttribute('enabled', true); - $log = $schedule - ->setAttribute('resourceUpdatedAt', $function['resourceUpdatedAt']) - ->setAttribute('schedule', $function['schedule']) - ->setAttribute('active', !empty($function->getAttribute('schedule') || !empty($schedule))); - $function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [ 'execute' => $execute, 'name' => $name, 'events' => $events, - 'scheduleId' => $log->getId(), 'schedule' => $schedule, 'scheduleUpdatedAt' => DateTime::now(), 'timeout' => $timeout, @@ -475,6 +469,15 @@ App::put('/v1/functions/:functionId') 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); + $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + + $log + ->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']) + ->setAttribute('schedule', $function['schedule']) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + + $dbForConsole->updateDocument('schedules', $log->getId(), $log); + $eventsInstance->setParam('functionId', $function->getId()); $response->dynamic($function, Response::MODEL_FUNCTION); @@ -527,9 +530,15 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') 'deployment' => $deployment->getId() ]))); - $log = $dbForProject->getDocument('schedules', $function['resourceId']); + $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); - $log->setAttribute('active', true); + $active = !empty($function->getAttribute('schedule')); + + if ($active) { + $log->setAttribute('resourceUpdatedAt', datetime::now()); + } + + $log->setAttribute('active', $active); Authorization::skip(function () use ($dbForConsole, $log) { $dbForConsole->updateDocument('schedules', $log->getId(), $log); @@ -574,7 +583,7 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } - $log = $dbForProject->getDocument('schedules', $function['resourceId']); + $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); $log ->setAttribute('resourceUpdatedAt', DateTime::now()) @@ -774,9 +783,19 @@ App::post('/v1/functions/:functionId/deployments') } } - $log = $dbForProject->getDocument('schedules', $function['resourceId']); + /** + * TODO Should we update also the function collection with the scheduleUpdatedAt attr? + */ - $log->setAttribute('active', true); + $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + + $active = !empty($function->getAttribute('schedule')); + + if ($active) { + $log->setAttribute('resourceUpdatedAt', datetime::now()); + } + + $log->setAttribute('active', $active); Authorization::skip(function () use ($dbForConsole, $log) { $dbForConsole->updateDocument('schedules', $log->getId(), $log); diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 4664c8c10f..b86d5a286d 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -1,5 +1,6 @@ task('schedule') ->desc('Function scheduler task') @@ -23,6 +26,7 @@ $cli Console::success(APP_NAME . ' Scheduler v1 has started'); $createQueue = function () use (&$functions, &$queue) { + $loadStart = \microtime(true); /** * Creating smaller functions list containing 5-min timeframe. */ @@ -34,6 +38,9 @@ $cli $queue[$next][$function['resourceId']] = $function; } } + $loadEnd = \microtime(true); + Console::error("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); + }; $removeFromQueue = function ($scheduleId) use (&$queue) { @@ -49,12 +56,13 @@ $cli $dbForConsole = getConsoleDB(); - $count = 0; - $limit = 50; + $limit = 200; $sum = $limit; $functions = []; $queue = []; - + $count = 0; + $loadStart = \microtime(true); + $total = 0; /** * Initial run fill $functions list */ @@ -63,17 +71,24 @@ $cli Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('resourceType', ['function']), Query::equal('active', [true]), - Query::limit($limit) + Query::offset($count * $limit), + Query::limit($limit), ]); $sum = count($results); + + $total = $total + $sum; foreach ($results as $document) { $functions[$document['resourceId']] = $document; - $count++; } + $count++; } + $loadEnd = \microtime(true); + Console::error("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds"); + $createQueue(); + $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_VALIDATION_TIMER); Co\run( @@ -88,12 +103,13 @@ $cli /** * Updating functions list from DB. */ - while ($sum === $limit) { + while (!empty($sum)) { $results = $dbForConsole->find('schedules', [ Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('resourceType', ['function']), Query::greaterThan('resourceUpdatedAt', $lastUpdate), - Query::limit($limit) + Query::limit($limit), + Query::offset($count * $limit), ]); $sum = count($results); foreach ($results as $document) { @@ -107,8 +123,8 @@ $cli $functions[$document['resourceId']] = $document; } $removeFromQueue($document['resourceId']); - $count++; } + $count++; } $lastUpdate = DateTime::now(); diff --git a/docker-compose.yml b/docker-compose.yml index 0dac1dc094..045e70dec6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev - - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database + #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis From 9ed6ed69fb60ab1ccc6dad66bd844724b6e034b8 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 7 Nov 2022 13:22:52 +0200 Subject: [PATCH 06/65] queue --- app/tasks/schedule.php | 45 +++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index b86d5a286d..2fc7acd1e9 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -1,4 +1,5 @@ task('schedule') @@ -40,7 +55,6 @@ $cli } $loadEnd = \microtime(true); Console::error("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); - }; $removeFromQueue = function ($scheduleId) use (&$queue) { @@ -54,7 +68,6 @@ $cli } }; - $dbForConsole = getConsoleDB(); $limit = 200; $sum = $limit; @@ -76,7 +89,6 @@ $cli ]); $sum = count($results); - $total = $total + $sum; foreach ($results as $document) { $functions[$document['resourceId']] = $document; @@ -96,8 +108,10 @@ $cli Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $count = 0; - $limit = 50; + $limit = 200; $sum = $limit; + $total = 0; + $timerStart = \microtime(true); Console::info("Update proc run at: $time last update was at $lastUpdate"); /** @@ -112,6 +126,7 @@ $cli Query::offset($count * $limit), ]); $sum = count($results); + $total = $total + $sum; foreach ($results as $document) { $org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null; $new = strtotime($document['resourceUpdatedAt']); @@ -126,24 +141,27 @@ $cli } $count++; } - $lastUpdate = DateTime::now(); $createQueue(); + $timerEnd = \microtime(true); + + Console::error("Update timer: {$total} functions where updated in " . ($timerStart - $timerEnd) . " seconds"); }); Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { + $timerStart = \microtime(true); $time = DateTime::now(); $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */ $now = (new \DateTime())->format('Y-m-d H:i:00.000'); Console::info("Enqueue proc run at: $time"); // Debug - foreach ($queue as $slot => $schedule) { - Console::log("Slot: $slot"); - foreach ($schedule as $function) { - Console::log("{$function['resourceId']} {$function['schedule']}"); - } - } + // foreach ($queue as $slot => $schedule) { + // Console::log("Slot: $slot"); + // foreach ($schedule as $function) { + // Console::log("{$function['resourceId']} {$function['schedule']}"); + // } + // } /** * Lopping time slots @@ -153,11 +171,12 @@ $cli if ($now === $slot) { foreach ($schedule as $function) { /** - * Enqueue function + * Enqueue function (here should be the Enqueue call */ Console::warning("Enqueueing :{$function['resourceId']}"); $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); + /** * If next schedule is in 5-min timeframe * and it was not removed or changed, re-enqueue the function. @@ -175,6 +194,8 @@ $cli unset($queue[$slot]); /** removing slot */ } } + $timerEnd = \microtime(true); + Console::error("Queue timer: finished in " . ($timerStart - $timerEnd) . " seconds"); }); } ); From 44092175213a575f67dc55723af4908b428d21d7 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 7 Nov 2022 14:20:02 +0200 Subject: [PATCH 07/65] queue --- app/tasks/schedule.php | 55 ++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 2fc7acd1e9..d556c30528 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -145,14 +145,14 @@ $cli $createQueue(); $timerEnd = \microtime(true); - Console::error("Update timer: {$total} functions where updated in " . ($timerStart - $timerEnd) . " seconds"); + Console::error("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); }); Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { $timerStart = \microtime(true); $time = DateTime::now(); $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */ - $now = (new \DateTime())->format('Y-m-d H:i:00.000'); + $slot = (new \DateTime())->format('Y-m-d H:i:00.000'); Console::info("Enqueue proc run at: $time"); // Debug @@ -163,39 +163,36 @@ $cli // } // } - /** - * Lopping time slots - */ + if (array_key_exists($slot, $queue)) { + $schedule = $queue[$slot]; + console::log("Number of function sent to worker (" . count($schedule)); - foreach ($queue as $slot => $schedule) { - if ($now === $slot) { - foreach ($schedule as $function) { - /** - * Enqueue function (here should be the Enqueue call - */ - Console::warning("Enqueueing :{$function['resourceId']}"); - $cron = new CronExpression($function['schedule']); - $next = DateTime::format($cron->getNextRunDate()); + foreach ($schedule as $function) { + /** + * Enqueue function (here should be the Enqueue call + */ + //Console::warning("Enqueueing :{$function['resourceId']}"); + $cron = new CronExpression($function['schedule']); + $next = DateTime::format($cron->getNextRunDate()); - /** - * If next schedule is in 5-min timeframe - * and it was not removed or changed, re-enqueue the function. - */ - if ( - $next < $timeFrame && - !empty($functions[$function['resourceId']] && - $function['schedule'] === $functions[$function['resourceId']]['schedule']) - ) { - Console::warning("re-enqueueing :{$function['resourceId']}"); - $queue[$next][$function['resourceId']] = $function; - } - unset($queue[$slot][$function['resourceId']]); /** removing function from slot */ + /** + * If next schedule is in 5-min timeframe + * and it was not removed or changed, re-enqueue the function. + */ + if ( + $next < $timeFrame && + !empty($functions[$function['resourceId']] && + $function['schedule'] === $functions[$function['resourceId']]['schedule']) + ) { + //Console::warning("re-enqueueing :{$function['resourceId']}"); + $queue[$next][$function['resourceId']] = $function; } - unset($queue[$slot]); /** removing slot */ + unset($queue[$slot][$function['resourceId']]); /** removing function from slot */ } + unset($queue[$slot]); /** removing slot */ } $timerEnd = \microtime(true); - Console::error("Queue timer: finished in " . ($timerStart - $timerEnd) . " seconds"); + Console::error("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds"); }); } ); From 3768912412b46557d354863c3b1f93fe90a85dd1 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 7 Nov 2022 16:39:42 +0200 Subject: [PATCH 08/65] replacing offset with pagination --- app/tasks/schedule.php | 65 ++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index d556c30528..2f58b9845d 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -69,62 +69,70 @@ $cli }; $dbForConsole = getConsoleDB(); - $limit = 200; + $limit = 10000; $sum = $limit; $functions = []; $queue = []; - $count = 0; - $loadStart = \microtime(true); $total = 0; - /** - * Initial run fill $functions list - */ + $loadStart = \microtime(true); + $latestDocument = null; + while ($sum === $limit) { - $results = $dbForConsole->find('schedules', [ + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('resourceType', ['function']), Query::equal('active', [true]), - Query::offset($count * $limit), - Query::limit($limit), - ]); + ])); $sum = count($results); $total = $total + $sum; foreach ($results as $document) { - $functions[$document['resourceId']] = $document; + $functions[$document['resourceId']] = [ + 'resourceId' => $document->getAttribute('resourceId'), + 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), + 'schedule' => $document->getAttribute('schedule'), + ]; } - $count++; + + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } $loadEnd = \microtime(true); Console::error("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds"); $createQueue(); - $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_VALIDATION_TIMER); + /** + * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. + */ Co\run( function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); - $count = 0; $limit = 200; $sum = $limit; $total = 0; + $latestDocument = null; $timerStart = \microtime(true); Console::info("Update proc run at: $time last update was at $lastUpdate"); - /** - * Updating functions list from DB. - */ - while (!empty($sum)) { - $results = $dbForConsole->find('schedules', [ + + while ($sum === $limit) { + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('resourceType', ['function']), Query::greaterThan('resourceUpdatedAt', $lastUpdate), - Query::limit($limit), - Query::offset($count * $limit), - ]); + ])); + $sum = count($results); $total = $total + $sum; foreach ($results as $document) { @@ -135,12 +143,18 @@ $cli unset($functions[$document['resourceId']]); } elseif ($new > $org) { Console::error("Updating: {$document['resourceId']}"); - $functions[$document['resourceId']] = $document; + $functions[$document['resourceId']] = [ + 'resourceId' => $document->getAttribute('resourceId'), + 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), + 'schedule' => $document->getAttribute('schedule'), + ]; } $removeFromQueue($document['resourceId']); } - $count++; + + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } + $lastUpdate = DateTime::now(); $createQueue(); $timerEnd = \microtime(true); @@ -148,6 +162,9 @@ $cli Console::error("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); }); + /** + * The timer sends to worker every 1 min and re-enqueue matched functions. + */ Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { $timerStart = \microtime(true); $time = DateTime::now(); From 711d8061a692f7c205d21195ddd3230febaa7f02 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 7 Nov 2022 17:30:22 +0200 Subject: [PATCH 09/65] replacing offset with pagination --- app/controllers/api/functions.php | 7 ++++++- app/tasks/schedule.php | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index c73fbb686c..98dd8fbd75 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -471,8 +471,13 @@ App::put('/v1/functions/:functionId') $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $active = !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')); + + if ($active) { + $log->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); + } + $log - ->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']) ->setAttribute('schedule', $function['schedule']) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 2f58b9845d..c5df617269 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -12,7 +12,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Query; use Swoole\Timer; -const FUNCTION_VALIDATION_TIMER = 180; //seconds +const FUNCTION_UPDATE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMER = 60; //seconds const ENQUEUE_TIME_FRAME = 60 * 5; // 5 min sleep(4); // Todo prevent PDOException @@ -61,7 +61,7 @@ $cli foreach ($queue as $slot => $schedule) { foreach ($schedule as $function) { if ($scheduleId === $function['resourceId']) { - Console::error("Unsetting :{$function['resourceId']} from queue slot $slot"); + Console::info("Unsetting :{$function['resourceId']} from queue slot $slot"); unset($queue[$slot][$function['resourceId']]); } } @@ -105,14 +105,14 @@ $cli Console::error("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds"); $createQueue(); - $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_VALIDATION_TIMER); + $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_UPDATE_TIMER); /** * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. */ Co\run( function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { - Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $limit = 200; $sum = $limit; @@ -139,10 +139,10 @@ $cli $org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null; $new = strtotime($document['resourceUpdatedAt']); if ($document['active'] === false) { - Console::error("Removing: {$document['resourceId']}"); + Console::info("Removing: {$document['resourceId']}"); unset($functions[$document['resourceId']]); } elseif ($new > $org) { - Console::error("Updating: {$document['resourceId']}"); + Console::info("Updating: {$document['resourceId']}"); $functions[$document['resourceId']] = [ 'resourceId' => $document->getAttribute('resourceId'), 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), From 3a651adea52295db3ed7039eb8ed7661560134dc Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 7 Nov 2022 17:42:08 +0200 Subject: [PATCH 10/65] replacing offset with pagination --- app/controllers/api/functions.php | 6 +++--- app/tasks/schedule.php | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 98dd8fbd75..bfb6ab1ae8 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -96,7 +96,7 @@ App::post('/v1/functions') 'resourceId' => $function->getId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), - 'schedule' => $function['schedule'], + 'schedule' => $function->getAttribute('schedule'), 'active' => false, ])) ); @@ -478,8 +478,8 @@ App::put('/v1/functions/:functionId') } $log - ->setAttribute('schedule', $function['schedule']) - ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', $active); $dbForConsole->updateDocument('schedules', $log->getId(), $log); diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index c5df617269..0134fe042c 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -1,7 +1,5 @@ Date: Tue, 8 Nov 2022 16:00:39 +0200 Subject: [PATCH 11/65] tidy up --- app/controllers/api/functions.php | 10 +++++---- app/tasks/schedule.php | 35 ++++++++++++------------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index bfb6ab1ae8..46296bee37 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -471,15 +471,17 @@ App::put('/v1/functions/:functionId') $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); - $active = !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')); - - if ($active) { + /** + * In case we want to clear the schedule + */ + if (!empty($function->getAttribute('deployment'))) { $log->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); } $log ->setAttribute('schedule', $function->getAttribute('schedule')) - ->setAttribute('active', $active); + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + $dbForConsole->updateDocument('schedules', $log->getId(), $log); diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 0134fe042c..a478267b8f 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -52,14 +52,14 @@ $cli } } $loadEnd = \microtime(true); - Console::error("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); + Console::success("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); }; $removeFromQueue = function ($scheduleId) use (&$queue) { foreach ($queue as $slot => $schedule) { foreach ($schedule as $function) { if ($scheduleId === $function['resourceId']) { - Console::info("Unsetting :{$function['resourceId']} from queue slot $slot"); + Console::error("Unsetting :{$function['resourceId']} from queue slot $slot"); unset($queue[$slot][$function['resourceId']]); } } @@ -91,8 +91,8 @@ $cli foreach ($results as $document) { $functions[$document['resourceId']] = [ 'resourceId' => $document->getAttribute('resourceId'), - 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), 'schedule' => $document->getAttribute('schedule'), + 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), ]; } @@ -100,8 +100,7 @@ $cli } $loadEnd = \microtime(true); - Console::error("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds"); - + Console::success("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds"); $createQueue(); $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_UPDATE_TIMER); @@ -112,13 +111,13 @@ $cli function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); - $limit = 200; + $limit = 1000; $sum = $limit; $total = 0; $latestDocument = null; $timerStart = \microtime(true); - Console::info("Update proc run at: $time last update was at $lastUpdate"); + Console::warning("Update proc started at: $time last update was at $lastUpdate"); while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; @@ -137,14 +136,14 @@ $cli $org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null; $new = strtotime($document['resourceUpdatedAt']); if ($document['active'] === false) { - Console::info("Removing: {$document['resourceId']}"); + Console::warning("Removing: {$document['resourceId']}"); unset($functions[$document['resourceId']]); } elseif ($new > $org) { - Console::info("Updating: {$document['resourceId']}"); + Console::warning("Updating: {$document['resourceId']}"); $functions[$document['resourceId']] = [ 'resourceId' => $document->getAttribute('resourceId'), - 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), 'schedule' => $document->getAttribute('schedule'), + 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), ]; } $removeFromQueue($document['resourceId']); @@ -157,7 +156,7 @@ $cli $createQueue(); $timerEnd = \microtime(true); - Console::error("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); + Console::warning("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); }); /** @@ -169,18 +168,11 @@ $cli $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */ $slot = (new \DateTime())->format('Y-m-d H:i:00.000'); - Console::info("Enqueue proc run at: $time"); - // Debug - // foreach ($queue as $slot => $schedule) { - // Console::log("Slot: $slot"); - // foreach ($schedule as $function) { - // Console::log("{$function['resourceId']} {$function['schedule']}"); - // } - // } + Console::info("Enqueue proc started at: $time"); if (array_key_exists($slot, $queue)) { $schedule = $queue[$slot]; - console::log("Number of function sent to worker (" . count($schedule)); + console::info(count($schedule) . " functions sent to worker for time slot " . $slot); foreach ($schedule as $function) { /** @@ -199,7 +191,6 @@ $cli !empty($functions[$function['resourceId']] && $function['schedule'] === $functions[$function['resourceId']]['schedule']) ) { - //Console::warning("re-enqueueing :{$function['resourceId']}"); $queue[$next][$function['resourceId']] = $function; } unset($queue[$slot][$function['resourceId']]); /** removing function from slot */ @@ -207,7 +198,7 @@ $cli unset($queue[$slot]); /** removing slot */ } $timerEnd = \microtime(true); - Console::error("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds"); + Console::info("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds"); }); } ); From 1761b77d0f50703920407d53ab4e001aece50b0e Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 9 Nov 2022 19:01:43 +0200 Subject: [PATCH 12/65] function worker --- app/controllers/api/functions.php | 37 +- app/init.php | 49 +- app/tasks/schedule.php | 20 +- app/worker.php | 64 ++ app/workers/functions.php | 612 +++++++++--------- composer.json | 3 +- composer.lock | 213 ++++-- .../Utopia/Response/Model/Execution.php | 1 + src/Appwrite/Utopia/Response/Model/Func.php | 12 - 9 files changed, 625 insertions(+), 386 deletions(-) create mode 100644 app/worker.php diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 46296bee37..5b969e63d8 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -5,7 +5,6 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\Func; use Appwrite\Event\Validator\Event as ValidatorEvent; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\CustomId; @@ -14,6 +13,7 @@ use Utopia\Database\Permission; use Utopia\Database\Role; use Utopia\Database\Validator\UID; use Appwrite\Usage\Stats; +use Utopia\Pools\Group; use Utopia\Storage\Device; use Utopia\Storage\Validator\File; use Utopia\Storage\Validator\FileExt; @@ -41,6 +41,7 @@ use Utopia\CLI\Console; use Utopia\Database\Validator\Roles; use Utopia\Validator\Boolean; use Utopia\Database\Exception\Duplicate as DuplicateException; +use Utopia\Queue\Client as queue; include_once __DIR__ . '/../shared/api.php'; @@ -1040,8 +1041,6 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') $response->noContent(); }); - - App::post('/v1/functions/:functionId/executions') ->groups(['api', 'functions']) ->desc('Create Execution') @@ -1067,7 +1066,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('events') ->inject('usage') ->inject('mode') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode) { + ->inject('pools') + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1155,17 +1155,18 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $event = new Func(); - $event - ->setType('http') - ->setExecution($execution) - ->setFunction($function) - ->setData($data) - ->setJWT($jwt) - ->setProject($project) - ->setUser($user); - - $event->trigger(); + $queue = new queue(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); + $queue->enqueue([ + 'type' => 'http', + 'value' => [ + 'type' => 'http', + 'execution' => $execution, + 'function' => $function, + 'data' => $data, + 'jwt' => $jwt, + 'project' => $project, + 'user' => $user + ]]); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1198,11 +1199,11 @@ App::post('/v1/functions/:functionId/executions') deploymentId: $deployment->getId(), path: $build->getAttribute('outputPath', ''), vars: $vars, - data: $data, entrypoint: $deployment->getAttribute('entrypoint', ''), + data: $data, runtime: $function->getAttribute('runtime', ''), - timeout: $function->getAttribute('timeout', 0), - baseImage: $runtime['image'] + baseImage: $runtime['image'], + timeout: $function->getAttribute('timeout', 0) ); /** Update execution status */ diff --git a/app/init.php b/app/init.php index a08dcfd137..4500526583 100644 --- a/app/init.php +++ b/app/init.php @@ -38,6 +38,8 @@ use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; use Appwrite\URL\URL as AppwriteURL; +use Utopia\Queue\Client as SyncOut; +use Utopia\Queue\Connection\Redis as QueueRedis; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; @@ -75,6 +77,7 @@ use Ahc\Jwt\JWTException; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; +use Utopia\Queue; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -497,6 +500,7 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); + $register->set('pools', function () { $group = new Group(); @@ -522,30 +526,35 @@ $register->set('pools', function () { 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], + 'useResource' => true, ], 'database' => [ 'type' => 'database', 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], + 'useResource' => true, ], 'queue' => [ 'type' => 'queue', 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], + 'useResource' => false, ], 'pubsub' => [ 'type' => 'pubsub', 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], + 'useResource' => true, ], 'cache' => [ 'type' => 'cache', 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), 'multiple' => true, 'schemes' => ['redis'], + 'useResource' => true, ], ]; @@ -554,6 +563,7 @@ $register->set('pools', function () { $dsns = $connection['dsns'] ?? ''; $multipe = $connection['multiple'] ?? false; $schemes = $connection['schemes'] ?? []; + $useResource = $connection['useResource'] ?? true; $config = []; $dsns = explode(',', $connection['dsns'] ?? ''); @@ -576,7 +586,7 @@ $register->set('pools', function () { $dsnScheme = $dsn->getScheme(); $dsnDatabase = $dsn->getDatabase(); - if (!in_array($dsnScheme, $schemes)) { + if (!in_array($dsnScheme, $schemes) && $useResource) { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); } @@ -635,13 +645,15 @@ $register->set('pools', function () { }; $adapter->setDefaultDatabase($dsn->getDatabase()); - - break; - case 'queue': - $adapter = $resource(); break; case 'pubsub': + break; $adapter = $resource(); + case 'queue': + $adapter = match ($dsn->getScheme()) { + 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), + default => 'bla' + }; break; case 'cache': $adapter = match ($dsn->getScheme()) { @@ -664,12 +676,6 @@ $register->set('pools', function () { Config::setParam('pools-' . $key, $config); } - try { - $group->fill(); - } catch (\Throwable $th) { - Console::error('Connection failure: ' . $th->getMessage()); - } - return $group; }); $register->set('influxdb', function () { @@ -847,8 +853,7 @@ App::setResource('messaging', fn() => new Phone()); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); - -App::setResource('clients', function ($request, $console, $project) { +App::setResource('clients', function ($request, $console, $project) use ($register) { $console->setAttribute('platforms', [ // Always allow current host '$collection' => ID::custom('platforms'), 'name' => 'Current Host', @@ -1024,6 +1029,24 @@ App::setResource('console', function () { ]); }, []); +App::setResource('queue', function () { + + $fallbackForRedis = AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); + + $connection = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); + + $dsns = explode(',', $connection ?? ''); + $dsn = explode('=', $dsns[0]); + $dsn = $dsn[1] ?? ''; + return new DSN($dsn); +}, []); + App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index a478267b8f..f370d7da41 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -3,12 +3,14 @@ global $cli; global $register; +use Appwrite\Event\Event; use Cron\CronExpression; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\DateTime; use Utopia\Database\Query; use Swoole\Timer; +use Utopia\Queue\Client as worker; const FUNCTION_UPDATE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMER = 60; //seconds @@ -108,7 +110,7 @@ $cli * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. */ Co\run( - function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + function () use ($register, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $limit = 1000; @@ -162,7 +164,7 @@ $cli /** * The timer sends to worker every 1 min and re-enqueue matched functions. */ - Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { + Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($register, $dbForConsole, &$functions, &$queue) { $timerStart = \microtime(true); $time = DateTime::now(); $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */ @@ -175,9 +177,17 @@ $cli console::info(count($schedule) . " functions sent to worker for time slot " . $slot); foreach ($schedule as $function) { - /** - * Enqueue function (here should be the Enqueue call - */ + $pools = $register->get('pools'); + $worker = new worker(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); + $project = $dbForConsole->getDocument('projects', $function['projectId']); + $worker + ->enqueue([ + 'type' => 'schedule', + 'value' => [ + 'project' => $project, + 'function' => getProjectDB($project)->getDocument('functions', $function['projectId']), + ] + ]); //Console::warning("Enqueueing :{$function['resourceId']}"); $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); diff --git a/app/worker.php b/app/worker.php new file mode 100644 index 0000000000..eaef650d99 --- /dev/null +++ b/app/worker.php @@ -0,0 +1,64 @@ + $register); + +Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { + $pools = $register->get('pools'); + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('console'); + + return $database; +}, ['cache', 'register']); + +Server::setResource('cache', function (Registry $register) { + $pools = $register->get('pools'); + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['register']); + +App::setResource('logger', function ($register) { + return $register->get('logger'); +}, ['register']); + + +$pools = $register->get('pools'); +$client = $pools->get('queue')->pop()->getResource(); + + +$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); +$workerNumber = 1; + +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); diff --git a/app/workers/functions.php b/app/workers/functions.php index 1ba4b6575b..47d36e4ce2 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -1,100 +1,315 @@ getId(); + $deploymentId = $function->getAttribute('deployment', ''); + + /** Check if deployment exists */ + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if ($deployment->getAttribute('resourceId') !== $functionId) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); } - public function init(): void - { - $this->executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); } - public function run(): void - { - $type = $this->args['type'] ?? ''; - $events = $this->args['events'] ?? []; - $project = new Document($this->args['project'] ?? []); - $user = new Document($this->args['user'] ?? []); - $payload = json_encode($this->args['payload'] ?? []); + /** Check if build has exists */ + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + if ($build->isEmpty()) { + throw new Exception('Build not found', 404); + } + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready', 400); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready', 400); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $carry[$var->getAttribute('key')] = $var->getAttribute('value'); + return $carry; + }, []); + + /** Collect environment variables */ + $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_ID' => $functionId, + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, + 'APPWRITE_FUNCTION_TRIGGER' => $trigger, + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_FUNCTION_EVENT' => $event ?? '', + 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', + 'APPWRITE_FUNCTION_DATA' => $data ?? '', + 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', + 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', + ]); + + /** Execute function */ + try { + $executionResponse = $executor->createExecution( + projectId: $project->getId(), + deploymentId: $deploymentId, + path: $build->getAttribute('outputPath', ''), + vars: $vars, + entrypoint: $deployment->getAttribute('entrypoint', ''), + data: $vars['APPWRITE_FUNCTION_DATA'] ?? '', + runtime: $function->getAttribute('runtime', ''), + baseImage: $runtime['image'], + timeout: $function->getAttribute('timeout', 0) + ); + + /** Update execution status */ + $execution + ->setAttribute('status', $executionResponse['status']) + ->setAttribute('statusCode', $executionResponse['statusCode']) + ->setAttribute('response', $executionResponse['response']) + ->setAttribute('stdout', $executionResponse['stdout']) + ->setAttribute('stderr', $executionResponse['stderr']) + ->setAttribute('duration', $executionResponse['duration']); + } catch (\Throwable $th) { + $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); + $execution + ->setAttribute('duration', (float)$interval->format('%s.%f')) + ->setAttribute('status', 'failed') + ->setAttribute('statusCode', $th->getCode()) + ->setAttribute('stderr', $th->getMessage()); + Console::error($th->getMessage()); + } + + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + /** Trigger Webhook */ + $executionModel = new Execution(); + $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); + $executionUpdate + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) + ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) + ->trigger(); + + /** Trigger Functions */ + $executionUpdate + ->setClass(Event::FUNCTIONS_CLASS_NAME) + ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + ->trigger(); + + /** Trigger realtime event */ + $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ + 'functionId' => $function->getId(), + 'executionId' => $execution->getId() + ]); + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $execution + ); + Realtime::send( + projectId: 'console', + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + Realtime::send( + projectId: $project->getId(), + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + /** Update usage stats */ + global $register; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $statsd = $register->get('statsd'); + $usage = new Stats($statsd); + $usage + ->setParam('projectId', $project->getId()) + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')) + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit(); + } +}; + +$adapter = new Queue\Adapter\Swoole($client, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); +$server = new Queue\Server($adapter); + +$server->job() + ->inject('message') + ->inject('dbForProject') + ->action(function (Message $message, Database $dbForProject) use ($execute) { + $args = $message->getPayload()['value'] ?? []; + $type = $message->getPayload()['type'] ?? ''; + $events = $args['events'] ?? []; + $project = new Document($args['project'] ?? []); + $user = new Document($args['user'] ?? []); + // Where $payload comes from + $payload = json_encode($args['payload'] ?? []); if ($project->getId() === 'console') { return; } - $database = $this->getProjectDB($project); - /** * Handle Event execution. */ if (!empty($events)) { $limit = 30; - $sum = 30; - $offset = 0; - $functions = []; - /** @var Document[] $functions */ + $sum = $limit; + $total = 0; + $latestDocument = null; - while ($sum >= $limit) { - $functions = $database->find('functions', [ - Query::limit($limit), - Query::offset($offset), - Query::orderAsc('name'), - ]); - $sum = \count($functions); - $offset = $offset + $limit; + while ($sum === $limit) { + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForProject->find('functions', \array_merge($paginationQueries, [ + Query::orderAsc('name') + ])); + + $sum = count($results); + $total = $total + $sum; Console::log('Fetched ' . $sum . ' functions...'); - foreach ($functions as $function) { + foreach ($results as $function) { if (!array_intersect($events, $function->getAttribute('events', []))) { continue; } Console::success('Iterating function: ' . $function->getAttribute('name')); - $this->execute( - project: $project, - function: $function, - dbForProject: $database, - trigger: 'event', - // Pass first, most verbose event pattern - event: $events[0], - eventData: $payload, - user: $user - ); + // As event, pass first, most verbose event pattern + call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null); Console::success('Triggered function: ' . $events[0]); } + + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } return; @@ -103,33 +318,20 @@ class FunctionsV1 extends Worker /** * Handle Schedule and HTTP execution. */ - $user = new Document($this->args['user'] ?? []); - $project = new Document($this->args['project'] ?? []); - $execution = new Document($this->args['execution'] ?? []); - $function = new Document($this->args['function'] ?? []); + $user = new Document($args['user'] ?? []); + $project = new Document($args['project'] ?? []); + $execution = new Document($args['execution'] ?? []); + $function = new Document($args['function'] ?? []); switch ($type) { case 'http': - $jwt = $this->args['jwt'] ?? ''; - $data = $this->args['data'] ?? ''; - - $function = $database->getDocument('functions', $execution->getAttribute('functionId')); - - $this->execute( - project: $project, - function: $function, - dbForProject: $database, - executionId: $execution->getId(), - trigger: 'http', - data: $data, - user: $user, - jwt: $jwt - ); - + $jwt = $args['jwt'] ?? ''; + $data = $args['data'] ?? ''; + $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); + call_user_func($execute, $project, $function, $dbForProject, 'http', $execution->getId(), null, null, $data, $user, $jwt); break; case 'schedule': - $functionOriginal = $function; /* * 1. Get Original Task * 2. Check for updates @@ -143,242 +345,54 @@ class FunctionsV1 extends Worker * If error count bigger than allowed change status to pause */ - // Reschedule - $function = $database->getDocument('functions', $function->getId()); - - if (empty($function->getId())) { - throw new Exception('Function not found (' . $function->getId() . ')'); - } - - if ($functionOriginal->getAttribute('schedule') !== $function->getAttribute('schedule')) { // Schedule has changed from previous run, ignore this run. - return; - } - - if ($functionOriginal->getAttribute('scheduleUpdatedAt') !== $function->getAttribute('scheduleUpdatedAt')) { // Double execution due to rapid cron changes, ignore this run. - return; - } - - $cron = new CronExpression($function->getAttribute('schedule')); - $next = DateTime::format($cron->getNextRunDate()); - - $function = $function - ->setAttribute('scheduleNext', $next) - ->setAttribute('schedulePrevious', DateTime::now()); - - $function = $database->updateDocument( - 'functions', - $function->getId(), - $function - ); - - $reschedule = new Func(); - $reschedule - ->setFunction($function) - ->setType('schedule') - ->setUser($user) - ->setProject($project) - ->schedule(new \DateTime($next)); - ; - - $this->execute( - project: $project, - function: $function, - dbForProject: $database, - trigger: 'schedule' - ); - + call_user_func($execute, $project, $function, $dbForProject, 'schedule', null, null, null, null, null, null); break; } - } + }); - private function execute( - Document $project, - Document $function, - Database $dbForProject, - string $trigger, - string $executionId = null, - string $event = null, - string $eventData = null, - string $data = null, - ?Document $user = null, - string $jwt = null - ) { +$server + ->error() + ->inject('error') + ->inject('logger') + ->inject('register') + ->action(function ($error, $logger, $register) { - $user ??= new Document(); - $functionId = $function->getId(); - $deploymentId = $function->getAttribute('deployment', ''); + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - /** Check if deployment exists */ - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - if ($deployment->getAttribute('resourceId') !== $functionId) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + if ($error instanceof PDOException) { + throw $error; } - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + if ($error->getCode() >= 500 || $error->getCode() === 0) { + $log = new Log(); + + $log->setNamespace("appwrite-worker"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + $log->setAction('appwrite-worker-functions'); + $log->addTag('verboseType', get_class($error)); + $log->addTag('code', $error->getCode()); + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + $log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $logger->addLog($log); } - /** Check if build has exists */ - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - if ($build->isEmpty()) { - throw new Exception('Build not found', 404); - } + Console::error('[Error] Type: ' . get_class($error)); + Console::error('[Error] Message: ' . $error->getMessage()); + Console::error('[Error] File: ' . $error->getFile()); + Console::error('[Error] Line: ' . $error->getLine()); - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready', 400); - } + $register->get('pools')->reclaim(); + }); - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); - - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); - } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { - $carry[$var->getAttribute('key')] = $var->getAttribute('value'); - return $carry; - }, []); - - /** Collect environment variables */ - $vars = \array_merge($vars, [ - 'APPWRITE_FUNCTION_ID' => $functionId, - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, - 'APPWRITE_FUNCTION_TRIGGER' => $trigger, - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_EVENT' => $event ?? '', - 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', - 'APPWRITE_FUNCTION_DATA' => $data ?? '', - 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', - 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', - ]); - - /** Execute function */ - try { - $executionResponse = $this->executor->createExecution( - projectId: $project->getId(), - deploymentId: $deploymentId, - path: $build->getAttribute('outputPath', ''), - vars: $vars, - entrypoint: $deployment->getAttribute('entrypoint', ''), - data: $vars['APPWRITE_FUNCTION_DATA'] ?? '', - runtime: $function->getAttribute('runtime', ''), - timeout: $function->getAttribute('timeout', 0), - baseImage: $runtime['image'] - ); - - /** Update execution status */ - $execution - ->setAttribute('status', $executionResponse['status']) - ->setAttribute('statusCode', $executionResponse['statusCode']) - ->setAttribute('response', $executionResponse['response']) - ->setAttribute('stdout', $executionResponse['stdout']) - ->setAttribute('stderr', $executionResponse['stderr']) - ->setAttribute('duration', $executionResponse['duration']); - } catch (\Throwable $th) { - $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); - $execution - ->setAttribute('duration', (float)$interval->format('%s.%f')) - ->setAttribute('status', 'failed') - ->setAttribute('statusCode', $th->getCode()) - ->setAttribute('stderr', $th->getMessage()); - Console::error($th->getMessage()); - } - - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - /** Trigger Webhook */ - $executionModel = new Execution(); - $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); - $executionUpdate - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) - ->trigger(); - - /** Trigger Functions */ - $executionUpdate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) - ->trigger(); - - /** Trigger realtime event */ - $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ - 'functionId' => $function->getId(), - 'executionId' => $execution->getId() - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $execution - ); - Realtime::send( - projectId: 'console', - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - /** Update usage stats */ - global $register; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $statsd = $register->get('statsd'); - $usage = new Stats($statsd); - $usage - ->setParam('projectId', $project->getId()) - ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')) - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ->submit(); - } - } - - public function shutdown(): void - { - } -} +$server->workerStart(); +$server->start(); diff --git a/composer.json b/composer.json index e09bec422e..6843ef6894 100644 --- a/composer.json +++ b/composer.json @@ -61,7 +61,8 @@ "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.6.*", - "utopia-php/pools": "0.1.*", + "utopia-php/queue": "0.4.0", + "utopia-php/pools": "dev-upgrade-cli as 0.2.0", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index 28d48b7f3d..6c6d129458 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f3beee3a829a19e53b311052111bde2c", + "content-hash": "a744959294e219fff6ea9c17f9fb0705", "packages": [ { "name": "adhocore/jwt", @@ -115,15 +115,15 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.11.0", + "version": "0.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "547fc026e11c0946846a8ac690898f5bf53be101" + "reference": "9d74a477ba3333cbcfac565c46fcf19606b7b603" }, "require": { "php": ">=8.0", - "utopia-php/system": "0.4.*" + "utopia-php/system": "0.6.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -154,7 +154,7 @@ "php", "runtimes" ], - "time": "2022-08-15T14:03:36+00:00" + "time": "2022-11-07T16:45:52+00:00" }, { "name": "chillerlan/php-qrcode", @@ -300,16 +300,16 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.13.1", + "version": "v1.14.0", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49" + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/dccc8a46586475075fbb012d8bd523b8a938c2dc", + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc", "shasum": "" }, "require": { @@ -341,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1" + "source": "https://github.com/colinmollenhour/credis/tree/v1.14.0" }, - "time": "2022-06-20T22:56:59+00:00" + "time": "2022-11-09T01:18:39+00:00" }, { "name": "composer/package-versions-deprecated", @@ -693,16 +693,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.1", + "version": "2.4.3", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379" + "reference": "67c26b443f348a51926030c83481b85718457d3d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/69568e4293f4fa993f3b0e51c9723e1e17c41379", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", + "reference": "67c26b443f348a51926030c83481b85718457d3d", "shasum": "" }, "require": { @@ -792,7 +792,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.1" + "source": "https://github.com/guzzle/psr7/tree/2.4.3" }, "funding": [ { @@ -808,7 +808,7 @@ "type": "tidelift" } ], - "time": "2022-08-28T14:45:39+00:00" + "time": "2022-10-26T14:07:24+00:00" }, { "name": "influxdb/influxdb-php", @@ -931,6 +931,72 @@ }, "time": "2021-02-04T16:20:16+00:00" }, + { + "name": "laravel/pint", + "version": "v1.2.0", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.11.0", + "illuminate/view": "^9.27", + "laravel-zero/framework": "^9.1.3", + "mockery/mockery": "^1.5.0", + "nunomaduro/larastan": "^2.2", + "nunomaduro/termwind": "^1.14.0", + "pestphp/pest": "^1.22.1" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2022-09-13T15:07:15+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -2431,23 +2497,24 @@ }, { "name": "utopia-php/pools", - "version": "0.1.0", + "version": "dev-upgrade-cli", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805" + "reference": "88a2c1ed2badbfdf2787ce0a12def2c988fc1097" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/5a467a569a80aefc846a97dc195b4adc2fd71805", - "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/88a2c1ed2badbfdf2787ce0a12def2c988fc1097", + "reference": "88a2c1ed2badbfdf2787ce0a12def2c988fc1097", "shasum": "" }, "require": { "ext-mongodb": "*", "ext-pdo": "*", "ext-redis": "*", - "php": ">=8.0" + "php": ">=8.0", + "utopia-php/cli": "0.13.*" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -2478,9 +2545,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.1.0" + "source": "https://github.com/utopia-php/pools/tree/upgrade-cli" }, - "time": "2022-10-11T19:31:07+00:00" + "time": "2022-11-04T08:33:04+00:00" }, { "name": "utopia-php/preloader", @@ -2535,6 +2602,67 @@ }, "time": "2020-10-24T07:04:59+00:00" }, + { + "name": "utopia-php/queue", + "version": "0.4.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/queue.git", + "reference": "0cad4cf4231377aa6c67956b51ba1954e0d02166" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/0cad4cf4231377aa6c67956b51ba1954e0d02166", + "reference": "0cad4cf4231377aa6c67956b51ba1954e0d02166", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.13.*", + "utopia-php/framework": "0.*.*" + }, + "require-dev": { + "laravel/pint": "^0.2.3", + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.5.5", + "swoole/ide-helper": "4.8.8", + "workerman/workerman": "^4.0" + }, + "suggest": { + "ext-swoole": "Needed to support Swoole.", + "workerman/workerman": "Needed to support Workerman." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Queue\\": "src/Queue" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A powerful task queue.", + "keywords": [ + "Tasks", + "framework", + "php", + "queue", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/queue/issues", + "source": "https://github.com/utopia-php/queue/tree/0.4.0" + }, + "time": "2022-10-31T06:23:08+00:00" + }, { "name": "utopia-php/registry", "version": "0.5.0", @@ -2700,23 +2828,25 @@ }, { "name": "utopia-php/system", - "version": "0.4.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0" + "reference": "289c4327713deadc9c748b5317d248133a02f245" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0", + "url": "https://api.github.com/repos/utopia-php/system/zipball/289c4327713deadc9c748b5317d248133a02f245", + "reference": "289c4327713deadc9c748b5317d248133a02f245", "shasum": "" }, "require": { + "laravel/pint": "1.2.*", "php": ">=7.4" }, "require-dev": { "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", "vimeo/psalm": "4.0.1" }, "type": "library", @@ -2749,9 +2879,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.4.0" + "source": "https://github.com/utopia-php/system/tree/0.6.0" }, - "time": "2021-02-04T14:14:49+00:00" + "time": "2022-11-07T13:51:59+00:00" }, { "name": "utopia-php/websocket", @@ -3574,16 +3704,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.17", + "version": "9.2.18", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8" + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", "shasum": "" }, "require": { @@ -3639,7 +3769,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" }, "funding": [ { @@ -3647,7 +3777,7 @@ "type": "github" } ], - "time": "2022-08-30T12:24:04+00:00" + "time": "2022-10-27T13:35:33+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5402,11 +5532,18 @@ "version": "dev-feat-update-cache-lib", "alias": "0.26.1", "alias_normalized": "0.26.1.0" + }, + { + "package": "utopia-php/pools", + "version": "dev-upgrade-cli", + "alias": "0.2.0", + "alias_normalized": "0.2.0.0" } ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/database": 20 + "utopia-php/database": 20, + "utopia-php/pools": 20 }, "prefer-stable": false, "prefer-lowest": false, @@ -5431,5 +5568,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Utopia/Response/Model/Execution.php b/src/Appwrite/Utopia/Response/Model/Execution.php index 13011a24b7..987a140dfb 100644 --- a/src/Appwrite/Utopia/Response/Model/Execution.php +++ b/src/Appwrite/Utopia/Response/Model/Execution.php @@ -6,6 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use Utopia\Database\Role; + class Execution extends Model { public function __construct() diff --git a/src/Appwrite/Utopia/Response/Model/Func.php b/src/Appwrite/Utopia/Response/Model/Func.php index c7e69fff88..540b143876 100644 --- a/src/Appwrite/Utopia/Response/Model/Func.php +++ b/src/Appwrite/Utopia/Response/Model/Func.php @@ -81,18 +81,6 @@ class Func extends Model 'default' => '', 'example' => '5 4 * * *', ]) - ->addRule('scheduleNext', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Function\'s next scheduled execution time in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) - ->addRule('schedulePrevious', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Function\'s previous scheduled execution time in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) ->addRule('timeout', [ 'type' => self::TYPE_INTEGER, 'description' => 'Function execution timeout in seconds.', From aef565c8ffaf3e4760a34b7f8bd4c21816441750 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 10 Nov 2022 18:40:12 +0200 Subject: [PATCH 13/65] addressing comments --- app/config/collections.php | 2 +- app/controllers/api/functions.php | 40 +++++++++++++++---------------- app/tasks/schedule.php | 39 ++++++++++++++---------------- docker-compose.yml | 3 --- 4 files changed, 39 insertions(+), 45 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 345d802dc4..f9cdc52150 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2272,7 +2272,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('scheduleUpdatedAt'), // Used to fix duplicate executions bug. Can be removed once new queue library is used + '$id' => ID::custom('scheduleUpdatedAt'), 'type' => Database::VAR_DATETIME, 'format' => '', 'size' => 0, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 46296bee37..94a2e26deb 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -89,7 +89,7 @@ App::post('/v1/functions') 'search' => implode(' ', [$functionId, $name, $runtime]) ])); - $log = Authorization::skip( + $schedule = Authorization::skip( fn() => $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region 'resourceType' => 'function', @@ -101,7 +101,7 @@ App::post('/v1/functions') ])) ); - $function->setAttribute('scheduleId', $log->getId()); + $function->setAttribute('scheduleId', $schedule->getId()); $dbForProject->updateDocument('functions', $function->getId(), $function); $eventsInstance->setParam('functionId', $function->getId()); @@ -469,21 +469,21 @@ App::put('/v1/functions/:functionId') 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); /** * In case we want to clear the schedule */ if (!empty($function->getAttribute('deployment'))) { - $log->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); + $schedule->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); } - $log + $schedule ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); $eventsInstance->setParam('functionId', $function->getId()); @@ -537,18 +537,18 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') 'deployment' => $deployment->getId() ]))); - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); $active = !empty($function->getAttribute('schedule')); if ($active) { - $log->setAttribute('resourceUpdatedAt', datetime::now()); + $schedule->setAttribute('resourceUpdatedAt', datetime::now()); } - $log->setAttribute('active', $active); + $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $events @@ -590,15 +590,15 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); - $log + $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('active', false) ; - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $deletes @@ -794,18 +794,18 @@ App::post('/v1/functions/:functionId/deployments') * TODO Should we update also the function collection with the scheduleUpdatedAt attr? */ - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); $active = !empty($function->getAttribute('schedule')); if ($active) { - $log->setAttribute('resourceUpdatedAt', datetime::now()); + $schedule->setAttribute('resourceUpdatedAt', datetime::now()); } - $log->setAttribute('active', $active); + $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $metadata = null; diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index a478267b8f..0e41d56b5c 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -12,20 +12,19 @@ use Swoole\Timer; const FUNCTION_UPDATE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMER = 60; //seconds -const ENQUEUE_TIME_FRAME = 60 * 5; // 5 min -sleep(4); // Todo prevent PDOException - +const FUNCTION_ENQUEUE_TIMEFRAME = 60 * 5; // 5 min +sleep(4); /** - * 1. first load from db with limit+offset --line 82-- - * 2. creating a 5-min offset array ($queue) --line 102-- + * 1. first load from db with limit+offset + * 2. creating a 5-min offset array ($queue) * 3. First timer runs every minute, looping over $queue time slots (each slot is 1-min delta) * if the function matches the current minute it should be dispatched to the functions worker. * Then another translation is made to the cron pattern if it is in the next 5-min window - * it is assigned again to the $queue. --line 172--. + * it is assigned again to the $queue. . * 4. Second timer runs every X min and updates the $functions (large) list. * The query fetches only functions that [resourceUpdatedAt] attr changed from the - * last time the timer that was fired (X min) --line 120-- + * last time the timer that was fired (X min) * If the function was deleted it is unsets from the list ($functions) and the $queue. * In the end of the timer the $queue is created again. * @@ -43,7 +42,7 @@ $cli /** * Creating smaller functions list containing 5-min timeframe. */ - $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); + $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMEFRAME); foreach ($functions as $function) { $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); @@ -107,8 +106,7 @@ $cli /** * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. */ - Co\run( - function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $limit = 1000; @@ -165,7 +163,7 @@ $cli Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { $timerStart = \microtime(true); $time = DateTime::now(); - $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */ + $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMEFRAME); $slot = (new \DateTime())->format('Y-m-d H:i:00.000'); Console::info("Enqueue proc started at: $time"); @@ -175,17 +173,16 @@ $cli console::info(count($schedule) . " functions sent to worker for time slot " . $slot); foreach ($schedule as $function) { - /** - * Enqueue function (here should be the Enqueue call - */ - //Console::warning("Enqueueing :{$function['resourceId']}"); + /** + * Enqueue function (here should be the Enqueue call + */ $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); - /** - * If next schedule is in 5-min timeframe - * and it was not removed or changed, re-enqueue the function. - */ + /** + * If next schedule is in 5-min timeframe + * and it was not removed or changed, re-enqueue the function. + */ if ( $next < $timeFrame && !empty($functions[$function['resourceId']] && @@ -200,6 +197,6 @@ $cli $timerEnd = \microtime(true); Console::info("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds"); }); - } - ); + + }); diff --git a/docker-compose.yml b/docker-compose.yml index 045e70dec6..b23aecde5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis @@ -346,7 +345,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis - mariadb @@ -584,7 +582,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis environment: From 3dc619f57db189557cdb197ee48d5b8f8a522782 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 10 Nov 2022 21:07:27 +0200 Subject: [PATCH 14/65] addressing comments --- app/tasks/schedule.php | 8 +++++--- composer.json | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 0e41d56b5c..d23314066e 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -15,6 +15,7 @@ const FUNCTION_ENQUEUE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMEFRAME = 60 * 5; // 5 min sleep(4); + /** * 1. first load from db with limit+offset * 2. creating a 5-min offset array ($queue) @@ -106,7 +107,8 @@ $cli /** * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. */ - + Co\run( + function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $limit = 1000; @@ -197,6 +199,6 @@ $cli $timerEnd = \microtime(true); Console::info("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds"); }); - - + } + ); }); diff --git a/composer.json b/composer.json index 1775a2bdf5..304ea2bebe 100644 --- a/composer.json +++ b/composer.json @@ -61,7 +61,7 @@ "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.6.*", - "utopia-php/pools": "dev-feat-optimize-filling as 0.3.0", + "utopia-php/pools": "0.4.0", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", From 9e3b9b8e59265b1a12a408776f063bb4708deb68 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 11 Nov 2022 10:21:53 +0200 Subject: [PATCH 15/65] addressing comments --- app/tasks/schedule.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index d23314066e..a4ec1c5c04 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -55,13 +55,11 @@ $cli Console::success("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); }; - $removeFromQueue = function ($scheduleId) use (&$queue) { + $removeFromQueue = function ($resourceId) use (&$queue) { foreach ($queue as $slot => $schedule) { - foreach ($schedule as $function) { - if ($scheduleId === $function['resourceId']) { - Console::error("Unsetting :{$function['resourceId']} from queue slot $slot"); - unset($queue[$slot][$function['resourceId']]); - } + if (array_key_exists($resourceId, $schedule)) { + Console::error("Unsetting :{$resourceId} from queue slot $slot"); + unset($queue[$slot][$resourceId]); } } }; From 134dba0b4354f56c214393f8df3c0385f8527b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 12 Nov 2022 14:35:42 +0000 Subject: [PATCH 16/65] Fix startup errors --- app/preload.php | 1 + app/tasks/schedule.php | 1 + app/worker.php | 25 ++++++++++++++++++++-- app/workers/builds.php | 3 +++ app/workers/functions.php | 2 +- composer.lock | 44 +++++++++++++++++++-------------------- 6 files changed, 51 insertions(+), 25 deletions(-) diff --git a/app/preload.php b/app/preload.php index 4935db3da4..e587bfaed5 100644 --- a/app/preload.php +++ b/app/preload.php @@ -36,6 +36,7 @@ foreach ( realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload + realpath(__DIR__ . '/../vendor/utopia-php/queue/src/Queue/Adapter/Workerman.php'), // TODO: remove workerman autoload ] as $key => $value ) { if ($value !== false) { diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index f370d7da41..121885b6c9 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -95,6 +95,7 @@ $cli 'resourceId' => $document->getAttribute('resourceId'), 'schedule' => $document->getAttribute('schedule'), 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), + 'projectId' => $document->getAttribute('projectId') ]; } diff --git a/app/worker.php b/app/worker.php index eaef650d99..bb35576172 100644 --- a/app/worker.php +++ b/app/worker.php @@ -9,14 +9,14 @@ use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; global $register; - - Server::setResource('register', fn() => $register); Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { @@ -33,6 +33,27 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) return $database; }, ['cache', 'register']); +Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) { + $args = $message->getPayload()['value'] ?? []; + $project = new Document($args['project'] ?? []); + + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } + + $pools = $register->get('pools'); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +}, ['cache', 'register', 'message', 'dbForConsole']); + Server::setResource('cache', function (Registry $register) { $pools = $register->get('pools'); $list = Config::getParam('pools-cache', []); diff --git a/app/workers/builds.php b/app/workers/builds.php index 235ad0da72..42831fb029 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -188,11 +188,14 @@ class BuildsV1 extends Worker } /** Update function schedule */ + // TODO: @Meldiron refactor scheduler here + /* $schedule = $function->getAttribute('schedule', ''); $cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null; $next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? DateTime::format($cron->getNextRunDate()) : null; $function->setAttribute('scheduleNext', $next); $function = $dbForProject->updateDocument('functions', $function->getId(), $function); + */ } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); diff --git a/app/workers/functions.php b/app/workers/functions.php index 47d36e4ce2..86fc2998b3 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -27,7 +27,7 @@ Authorization::setDefaultStatus(false); global $client; global $workerNumber; -$executor = new Executor(App::getEnv('_APP_FUNCTIONS_PROXY_HOST')); +$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); $execute = function ( Document $project, diff --git a/composer.lock b/composer.lock index 6c6d129458..48b4039b7f 100644 --- a/composer.lock +++ b/composer.lock @@ -1724,16 +1724,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace" + "reference": "7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/cfa0ae98841b9e461207c13ab093d76b0fa7bace", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936", + "reference": "7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936", "shasum": "" }, "require": { @@ -1742,7 +1742,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -1787,7 +1787,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.27.0" }, "funding": [ { @@ -1803,7 +1803,7 @@ "type": "tidelift" } ], - "time": "2022-05-10T07:21:04+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "utopia-php/abuse", @@ -5187,16 +5187,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a", + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a", "shasum": "" }, "require": { @@ -5211,7 +5211,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -5249,7 +5249,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0" }, "funding": [ { @@ -5265,20 +5265,20 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534", + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534", "shasum": "" }, "require": { @@ -5293,7 +5293,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -5332,7 +5332,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0" }, "funding": [ { @@ -5348,7 +5348,7 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "textalk/websocket", @@ -5568,5 +5568,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From 250ea93d3fe04db854258fac2b885e1042e8ca88 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 13 Nov 2022 15:14:00 +0200 Subject: [PATCH 17/65] some fixes --- app/tasks/schedule.php | 114 +++++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 28 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index a4ec1c5c04..ded7301208 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -7,12 +7,14 @@ use Cron\CronExpression; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\DateTime; +use Utopia\Database\Document; use Utopia\Database\Query; use Swoole\Timer; const FUNCTION_UPDATE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMEFRAME = 60 * 5; // 5 min +const FUNCTION_RESET_TIMER_TO = 50; // seconds sleep(4); @@ -38,7 +40,12 @@ $cli Console::title('Scheduler V1'); Console::success(APP_NAME . ' Scheduler v1 has started'); - $createQueue = function () use (&$functions, &$queue) { + $dbForConsole = getConsoleDB(); + + /** + * @return void + */ + $createQueue = function () use (&$functions, &$queue): void { $loadStart = \microtime(true); /** * Creating smaller functions list containing 5-min timeframe. @@ -47,24 +54,67 @@ $cli foreach ($functions as $function) { $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); + if ($next < $timeFrame) { $queue[$next][$function['resourceId']] = $function; } } $loadEnd = \microtime(true); Console::success("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); + //var_dump($queue); }; - $removeFromQueue = function ($resourceId) use (&$queue) { + /** + * @param string $id + * @param string $resourceId + * @return void + */ + $removeFromQueue = function (string $id, string $resourceId) use (&$queue, &$functions, $dbForConsole) { + if (array_key_exists($resourceId, $functions)) { + unset($functions[$resourceId]); + $dbForConsole->deleteDocument('schedules', $id); + Console::error("Removing :{$resourceId} from functions list"); + } + foreach ($queue as $slot => $schedule) { if (array_key_exists($resourceId, $schedule)) { - Console::error("Unsetting :{$resourceId} from queue slot $slot"); unset($queue[$slot][$resourceId]); + Console::error("Removing :{$resourceId} from queue slot $slot"); } } }; - $dbForConsole = getConsoleDB(); + /** + * @param string $resourceId + * @param array $update + * @return void + */ + $updateQueue = function (string $resourceId, array $update) use (&$queue, &$functions): void { + + $functions[$resourceId] = $update; + Console::error("Updating :{$resourceId} in functions list"); + + foreach ($queue as $slot => $schedule) { + if (array_key_exists($resourceId, $schedule)) { + $queue[$slot][$resourceId] = $update; + Console::error("Updating :{$resourceId} in queue slot $slot"); + } + } + }; + + /** + * @var Document $schedule + * @return array + */ + function getsSheduleAttributes(Document $schedule): array + { + return [ + 'resourceId' => $schedule->getAttribute('resourceId'), + 'schedule' => $schedule->getAttribute('schedule'), + 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), + ]; + } + $limit = 10000; $sum = $limit; $functions = []; @@ -87,11 +137,7 @@ $cli $sum = count($results); $total = $total + $sum; foreach ($results as $document) { - $functions[$document['resourceId']] = [ - 'resourceId' => $document->getAttribute('resourceId'), - 'schedule' => $document->getAttribute('schedule'), - 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), - ]; + $functions[$document['resourceId']] = getsSheduleAttributes($document); } $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; @@ -102,12 +148,20 @@ $cli $createQueue(); $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_UPDATE_TIMER); + do { + $second = time() % 60; + } while ($second < FUNCTION_RESET_TIMER_TO); + + $time = DateTime::now(); + Console::success("Starting timers at {$time}"); + + /** * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. */ Co\run( - function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { - Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + function () use ($updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $limit = 1000; $sum = $limit; @@ -115,7 +169,7 @@ $cli $latestDocument = null; $timerStart = \microtime(true); - Console::warning("Update proc started at: $time last update was at $lastUpdate"); + //Console::warning("Update proc started at: $time last update was at $lastUpdate"); while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; @@ -125,7 +179,7 @@ $cli $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('resourceType', ['function']), - Query::greaterThan('resourceUpdatedAt', $lastUpdate), + Query::greaterThanEqual('resourceUpdatedAt', $lastUpdate), ])); $sum = count($results); @@ -134,17 +188,12 @@ $cli $org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null; $new = strtotime($document['resourceUpdatedAt']); if ($document['active'] === false) { - Console::warning("Removing: {$document['resourceId']}"); - unset($functions[$document['resourceId']]); + //Console::warning("Removing: {$document['resourceId']}"); + $removeFromQueue($document->getId(), $document['resourceId']); } elseif ($new > $org) { - Console::warning("Updating: {$document['resourceId']}"); - $functions[$document['resourceId']] = [ - 'resourceId' => $document->getAttribute('resourceId'), - 'schedule' => $document->getAttribute('schedule'), - 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), - ]; + //Console::warning("Updating: {$document['resourceId']}"); + $updateQueue($document['resourceId'], getsSheduleAttributes($document)); } - $removeFromQueue($document['resourceId']); } $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; @@ -154,7 +203,7 @@ $cli $createQueue(); $timerEnd = \microtime(true); - Console::warning("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); + //Console::warning("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); }); /** @@ -165,17 +214,27 @@ $cli $time = DateTime::now(); $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMEFRAME); $slot = (new \DateTime())->format('Y-m-d H:i:00.000'); + $prepareStart = time(); Console::info("Enqueue proc started at: $time"); if (array_key_exists($slot, $queue)) { $schedule = $queue[$slot]; console::info(count($schedule) . " functions sent to worker for time slot " . $slot); + $totalPreparation = time() - $prepareStart; + + $wait = ((60 - FUNCTION_RESET_TIMER_TO) - $totalPreparation); + Console::info("Waiting for : {$wait} seconds"); + sleep($wait); + + $time = DateTime::now(); + Console::info("Start enqueueing at {$time}"); foreach ($schedule as $function) { - /** - * Enqueue function (here should be the Enqueue call - */ + if (empty($functions[$function['resourceId']])) { + continue; + } + $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); @@ -185,8 +244,7 @@ $cli */ if ( $next < $timeFrame && - !empty($functions[$function['resourceId']] && - $function['schedule'] === $functions[$function['resourceId']]['schedule']) + $function['schedule'] ?? [] === $functions[$function['resourceId']]['schedule'] ) { $queue[$next][$function['resourceId']] = $function; } From 5e7de81966ba0a4d6548df3552a4a6aed4053452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 14 Nov 2022 09:29:30 +0000 Subject: [PATCH 18/65] Improve schedule accuracy + simplify --- app/tasks/schedule.php | 218 ++++++++++++++-------------------------- app/workers/deletes.php | 5 + 2 files changed, 78 insertions(+), 145 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index ded7301208..33e030dbb7 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -11,98 +11,27 @@ use Utopia\Database\Document; use Utopia\Database\Query; use Swoole\Timer; -const FUNCTION_UPDATE_TIMER = 60; //seconds -const FUNCTION_ENQUEUE_TIMER = 60; //seconds -const FUNCTION_ENQUEUE_TIMEFRAME = 60 * 5; // 5 min -const FUNCTION_RESET_TIMER_TO = 50; // seconds - -sleep(4); +const FUNCTION_UPDATE_TIMER = 10; //seconds +const FUNCTION_ENQUEUE_TIMER = 10; //seconds /** - * 1. first load from db with limit+offset - * 2. creating a 5-min offset array ($queue) - * 3. First timer runs every minute, looping over $queue time slots (each slot is 1-min delta) - * if the function matches the current minute it should be dispatched to the functions worker. - * Then another translation is made to the cron pattern if it is in the next 5-min window - * it is assigned again to the $queue. . - * 4. Second timer runs every X min and updates the $functions (large) list. - * The query fetches only functions that [resourceUpdatedAt] attr changed from the - * last time the timer that was fired (X min) - * If the function was deleted it is unsets from the list ($functions) and the $queue. - * In the end of the timer the $queue is created again. - * + * 1. Load all documents from 'schedules' collection to create local copy + * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute + * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker. */ $cli ->task('schedule') ->desc('Function scheduler task') -->action(function () use ($register) { +->action(function () { Console::title('Scheduler V1'); Console::success(APP_NAME . ' Scheduler v1 has started'); $dbForConsole = getConsoleDB(); /** - * @return void - */ - $createQueue = function () use (&$functions, &$queue): void { - $loadStart = \microtime(true); - /** - * Creating smaller functions list containing 5-min timeframe. - */ - $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMEFRAME); - foreach ($functions as $function) { - $cron = new CronExpression($function['schedule']); - $next = DateTime::format($cron->getNextRunDate()); - - if ($next < $timeFrame) { - $queue[$next][$function['resourceId']] = $function; - } - } - $loadEnd = \microtime(true); - Console::success("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); - //var_dump($queue); - }; - - /** - * @param string $id - * @param string $resourceId - * @return void - */ - $removeFromQueue = function (string $id, string $resourceId) use (&$queue, &$functions, $dbForConsole) { - if (array_key_exists($resourceId, $functions)) { - unset($functions[$resourceId]); - $dbForConsole->deleteDocument('schedules', $id); - Console::error("Removing :{$resourceId} from functions list"); - } - - foreach ($queue as $slot => $schedule) { - if (array_key_exists($resourceId, $schedule)) { - unset($queue[$slot][$resourceId]); - Console::error("Removing :{$resourceId} from queue slot $slot"); - } - } - }; - - /** - * @param string $resourceId - * @param array $update - * @return void - */ - $updateQueue = function (string $resourceId, array $update) use (&$queue, &$functions): void { - - $functions[$resourceId] = $update; - Console::error("Updating :{$resourceId} in functions list"); - - foreach ($queue as $slot => $schedule) { - if (array_key_exists($resourceId, $schedule)) { - $queue[$slot][$resourceId] = $update; - Console::error("Updating :{$resourceId} in queue slot $slot"); - } - } - }; - - /** + * Extract only nessessary attributes to lower memory used. + * * @var Document $schedule * @return array */ @@ -115,10 +44,11 @@ $cli ]; } + $schedules = []; // Local copy of 'schedules' collection + $lastSyncUpdate = DateTime::now(); + $limit = 10000; $sum = $limit; - $functions = []; - $queue = []; $total = 0; $loadStart = \microtime(true); $latestDocument = null; @@ -137,39 +67,33 @@ $cli $sum = count($results); $total = $total + $sum; foreach ($results as $document) { - $functions[$document['resourceId']] = getsSheduleAttributes($document); + $schedules[$document['resourceId']] = getsSheduleAttributes($document); } $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } $loadEnd = \microtime(true); - Console::success("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds"); - $createQueue(); - $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_UPDATE_TIMER); - - do { - $second = time() % 60; - } while ($second < FUNCTION_RESET_TIMER_TO); + Console::success("{$total} schedules where loaded in " . ($loadEnd - $loadStart) . " seconds"); $time = DateTime::now(); - Console::success("Starting timers at {$time}"); + Console::success("Starting timers at {$time}"); - - /** - * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. - */ Co\run( - function () use ($updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { - Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + function () use ($dbForConsole, &$schedules, &$lastSyncUpdate) { + /** + * The timer synchronize $schedules copy with database collection. + */ + Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate) { $time = DateTime::now(); + $timerStart = \microtime(true); + $limit = 1000; $sum = $limit; $total = 0; $latestDocument = null; - $timerStart = \microtime(true); - //Console::warning("Update proc started at: $time last update was at $lastUpdate"); + Console::log("Sync tick: Running at $time"); while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; @@ -179,82 +103,86 @@ $cli $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ Query::equal('region', [App::getEnv('_APP_REGION')]), Query::equal('resourceType', ['function']), - Query::greaterThanEqual('resourceUpdatedAt', $lastUpdate), + Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); $sum = count($results); $total = $total + $sum; foreach ($results as $document) { - $org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null; + $localDocument = $schedules[$document['resourceId']] ?? null; + + $org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null; $new = strtotime($document['resourceUpdatedAt']); - if ($document['active'] === false) { - //Console::warning("Removing: {$document['resourceId']}"); - $removeFromQueue($document->getId(), $document['resourceId']); - } elseif ($new > $org) { - //Console::warning("Updating: {$document['resourceId']}"); - $updateQueue($document['resourceId'], getsSheduleAttributes($document)); + + if ($$document['active'] === false) { + Console::info("Removing: {$document['resourceId']}"); + unset($schedules[$document['resourceId']]); + } elseif ($new !== $org) { + Console::info("Updating: {$document['resourceId']}"); + $schedules[$document['resourceId']] = getsSheduleAttributes($document); } } $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } - $lastUpdate = DateTime::now(); - $createQueue(); + $lastSyncUpdate = $time; $timerEnd = \microtime(true); - //Console::warning("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); + Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); }); /** - * The timer sends to worker every 1 min and re-enqueue matched functions. + * The timer to prepare soon-to-execute schedules. */ - Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { + $lastEnqueueUpdate = null; + $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { $timerStart = \microtime(true); $time = DateTime::now(); - $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMEFRAME); - $slot = (new \DateTime())->format('Y-m-d H:i:00.000'); - $prepareStart = time(); - Console::info("Enqueue proc started at: $time"); + $enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate; + $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMER - $enqueueDiff); - if (array_key_exists($slot, $queue)) { - $schedule = $queue[$slot]; - console::info(count($schedule) . " functions sent to worker for time slot " . $slot); - $totalPreparation = time() - $prepareStart; + Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)"); - $wait = ((60 - FUNCTION_RESET_TIMER_TO) - $totalPreparation); - Console::info("Waiting for : {$wait} seconds"); - sleep($wait); + $total = 0; - $time = DateTime::now(); - Console::info("Start enqueueing at {$time}"); + foreach ($schedules as $key => $schedule) { + $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); + $next = DateTime::format($nextDate); - foreach ($schedule as $function) { - if (empty($functions[$function['resourceId']])) { - continue; - } + $currentTick = $next < $timeFrame; - $cron = new CronExpression($function['schedule']); - $next = DateTime::format($cron->getNextRunDate()); - - /** - * If next schedule is in 5-min timeframe - * and it was not removed or changed, re-enqueue the function. - */ - if ( - $next < $timeFrame && - $function['schedule'] ?? [] === $functions[$function['resourceId']]['schedule'] - ) { - $queue[$next][$function['resourceId']] = $function; - } - unset($queue[$slot][$function['resourceId']]); /** removing function from slot */ + if(!$currentTick) { + continue; } - unset($queue[$slot]); /** removing slot */ + + $total++; + + $promiseStart = \microtime(true); // in seconds + $executionStart = $nextDate->getTimestamp(); // in seconds + $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued + + \go(function() use ($executionSleep, $key, $schedules) { + \usleep($executionSleep * 1000000); // in microseconds + + // Ensure schedule was not deleted + if(!isset($schedules[$key])) { + return; + } + + Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue + }); } + $timerEnd = \microtime(true); - Console::info("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds"); - }); + $lastEnqueueUpdate = $timerStart; + Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); + }; + + Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); + $enqueueFunctions(); } ); }); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 9ef593dbb9..1980c6dc72 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -418,6 +418,11 @@ class DeletesV1 extends Worker $dbForProject = $this->getProjectDB($project); $functionId = $document->getId(); + /** + * Delete Schedule + */ + // TODO: DeleteDocument schedules collection + /** * Delete Variables */ From 83e19a37eec309ce9b1a34011262568310a095ec Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 14 Nov 2022 11:36:22 +0200 Subject: [PATCH 19/65] minor changes --- app/tasks/schedule.php | 75 +++++++++++++----------------------------- docker-compose.yml | 2 +- 2 files changed, 24 insertions(+), 53 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index ded7301208..1b301def5b 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -14,7 +14,6 @@ use Swoole\Timer; const FUNCTION_UPDATE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMEFRAME = 60 * 5; // 5 min -const FUNCTION_RESET_TIMER_TO = 50; // seconds sleep(4); @@ -32,7 +31,6 @@ sleep(4); * In the end of the timer the $queue is created again. * */ - $cli ->task('schedule') ->desc('Function scheduler task') @@ -59,9 +57,8 @@ $cli $queue[$next][$function['resourceId']] = $function; } } - $loadEnd = \microtime(true); - Console::success("Queue was built in " . ($loadEnd - $loadStart) . " seconds"); - //var_dump($queue); + + Console::success("Queue was built in " . (microtime(true) - $loadStart) . " seconds"); }; /** @@ -69,10 +66,9 @@ $cli * @param string $resourceId * @return void */ - $removeFromQueue = function (string $id, string $resourceId) use (&$queue, &$functions, $dbForConsole) { + $removeFromQueue = function (string $resourceId) use (&$queue, &$functions, $dbForConsole) { if (array_key_exists($resourceId, $functions)) { unset($functions[$resourceId]); - $dbForConsole->deleteDocument('schedules', $id); Console::error("Removing :{$resourceId} from functions list"); } @@ -106,14 +102,17 @@ $cli * @var Document $schedule * @return array */ - function getsSheduleAttributes(Document $schedule): array - { + $getSchedule = function (Document $schedule) use ($dbForConsole): array { + $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('schedule')); + return [ 'resourceId' => $schedule->getAttribute('resourceId'), 'schedule' => $schedule->getAttribute('schedule'), 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), + 'project' => $project, + //'function' => getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')) ]; - } + }; $limit = 10000; $sum = $limit; @@ -137,31 +136,19 @@ $cli $sum = count($results); $total = $total + $sum; foreach ($results as $document) { - $functions[$document['resourceId']] = getsSheduleAttributes($document); + $functions[$document['resourceId']] = $getSchedule($document); } $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } - $loadEnd = \microtime(true); - Console::success("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds"); + Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); $createQueue(); - $lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_UPDATE_TIMER); + $lastUpdate = DateTime::addSeconds(new \DateTime(), -600); // 10 min - do { - $second = time() % 60; - } while ($second < FUNCTION_RESET_TIMER_TO); - - $time = DateTime::now(); - Console::success("Starting timers at {$time}"); - - - /** - * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. - */ Co\run( - function () use ($updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { - Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + function () use ($getSchedule, $updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($getSchedule, $updateQueue, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $limit = 1000; $sum = $limit; @@ -169,7 +156,7 @@ $cli $latestDocument = null; $timerStart = \microtime(true); - //Console::warning("Update proc started at: $time last update was at $lastUpdate"); + Console::warning("Update proc started at: $time last update was at $lastUpdate"); while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; @@ -188,47 +175,29 @@ $cli $org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null; $new = strtotime($document['resourceUpdatedAt']); if ($document['active'] === false) { - //Console::warning("Removing: {$document['resourceId']}"); - $removeFromQueue($document->getId(), $document['resourceId']); + $removeFromQueue($document['resourceId']); } elseif ($new > $org) { - //Console::warning("Updating: {$document['resourceId']}"); - $updateQueue($document['resourceId'], getsSheduleAttributes($document)); + $updateQueue($document['resourceId'], $getSchedule($document)); } } - $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } $lastUpdate = DateTime::now(); $createQueue(); - $timerEnd = \microtime(true); - - //Console::warning("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds"); + Console::warning("Update timer: {$total} functions where updated in " . (microtime(true) - $timerStart) . " seconds"); }); - /** - * The timer sends to worker every 1 min and re-enqueue matched functions. - */ Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { $timerStart = \microtime(true); - $time = DateTime::now(); $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMEFRAME); $slot = (new \DateTime())->format('Y-m-d H:i:00.000'); - $prepareStart = time(); - Console::info("Enqueue proc started at: $time"); + Console::info("Enqueue proc started at: " . DateTime::now()); + $count = 0; if (array_key_exists($slot, $queue)) { $schedule = $queue[$slot]; - console::info(count($schedule) . " functions sent to worker for time slot " . $slot); - $totalPreparation = time() - $prepareStart; - - $wait = ((60 - FUNCTION_RESET_TIMER_TO) - $totalPreparation); - Console::info("Waiting for : {$wait} seconds"); - sleep($wait); - - $time = DateTime::now(); - Console::info("Start enqueueing at {$time}"); foreach ($schedule as $function) { if (empty($functions[$function['resourceId']])) { @@ -249,11 +218,13 @@ $cli $queue[$next][$function['resourceId']] = $function; } unset($queue[$slot][$function['resourceId']]); /** removing function from slot */ + $count++; } unset($queue[$slot]); /** removing slot */ } + $timerEnd = \microtime(true); - Console::info("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds"); + Console::info("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds with {$count} functions"); }); } ); diff --git a/docker-compose.yml b/docker-compose.yml index 7a2ca40b5e..5e38190668 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -764,7 +764,7 @@ services: mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p - container_name: mariadb + container_name: appwrite-mariadb <<: *x-logging networks: - appwrite From c2e8fc5733ecdf43365e692c200e757de338563f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 14 Nov 2022 10:00:46 +0000 Subject: [PATCH 20/65] Increase timer delay --- app/tasks/schedule.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 89899b5d67..6d742681b5 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -12,7 +12,7 @@ use Utopia\Database\Query; use Swoole\Timer; const FUNCTION_UPDATE_TIMER = 10; //seconds -const FUNCTION_ENQUEUE_TIMER = 10; //seconds +const FUNCTION_ENQUEUE_TIMER = 60; //seconds /** * 1. Load all documents from 'schedules' collection to create local copy @@ -76,11 +76,9 @@ $cli $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } - $loadEnd = \microtime(true); - Console::success("{$total} schedules where loaded in " . ($loadEnd - $loadStart) . " seconds"); + Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); - $time = DateTime::now(); - Console::success("Starting timers at {$time}"); + Console::success("Starting timers at " . DateTime::now()); Co\run( function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { From de8122c02fb5ddd0f98b9c262452c2e94b57d214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 14 Nov 2022 13:02:16 +0000 Subject: [PATCH 21/65] Share coroutines for executions --- app/tasks/schedule.php | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index 6d742681b5..4734191987 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -147,6 +147,8 @@ $cli $total = 0; + $delayedExecutions = []; // Group executions with same delay to share one coroutine + foreach ($schedules as $key => $schedule) { $cron = new CronExpression($schedule['schedule']); $nextDate = $cron->getNextRunDate(); @@ -164,15 +166,27 @@ $cli $executionStart = $nextDate->getTimestamp(); // in seconds $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued - \go(function() use ($executionSleep, $key, $schedules) { - \usleep($executionSleep * 1000000); // in microseconds + $delay = \intval($executionSleep); + + if(!isset($delayedExecutions[$delay])) { + $delayedExecutions[$delay] = []; + } - // Ensure schedule was not deleted - if(!isset($schedules[$key])) { - return; + $delayedExecutions[$delay][] = $key; + } + + foreach($delayedExecutions as $delay => $scheduleKeys) { + \go(function() use ($delay, $schedules, $scheduleKeys) { + \sleep($delay); // in seconds + + foreach($scheduleKeys as $scheduleKey) { + // Ensure schedule was not deleted + if(!isset($schedules[$scheduleKey])) { + return; + } + + Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue } - - Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue }); } From ff06e4b2385439209fbfd83d163187c0e5408697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 15 Nov 2022 10:15:57 +0000 Subject: [PATCH 22/65] Move schedule delete TODO --- app/tasks/maintenance.php | 2 ++ app/workers/deletes.php | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 1c16ca6911..c61c3dda3c 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -126,5 +126,7 @@ $cli notifyDeleteExpiredSessions(); renewCertificates($database); notifyDeleteCache($cacheRetention); + + // TODO: @Meldiron Every probably 24h, look for schedules with active=false, that doesnt have function anymore. Dlete such schedule }, $interval); }); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1980c6dc72..9ef593dbb9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -418,11 +418,6 @@ class DeletesV1 extends Worker $dbForProject = $this->getProjectDB($project); $functionId = $document->getId(); - /** - * Delete Schedule - */ - // TODO: DeleteDocument schedules collection - /** * Delete Variables */ From 22effdd88adb1feefd68418af325267fc6b0fb2a Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 11:37:07 +0100 Subject: [PATCH 23/65] Refactor schedule task to new syntax --- src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/schedule.php | 386 ++++++++++++----------- 2 files changed, 202 insertions(+), 186 deletions(-) diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 7f6a062ed4..2968a66b95 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -7,6 +7,7 @@ use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; +use Appwrite\Platform\Tasks\Schedule; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; @@ -28,6 +29,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) + ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php index 4734191987..e070206405 100644 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ b/src/Appwrite/Platform/Tasks/schedule.php @@ -1,202 +1,216 @@ task('schedule') -->desc('Function scheduler task') -->action(function () { - Console::title('Scheduler V1'); - Console::success(APP_NAME . ' Scheduler v1 has started'); - - $dbForConsole = getConsoleDB(); - - /** - * Extract only nessessary attributes to lower memory used. - * - * @var Document $schedule - * @return array - */ - $getSchedule = function (Document $schedule) use ($dbForConsole): array { - $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId')); - $function = getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')); - - return [ - 'resourceId' => $schedule->getAttribute('resourceId'), - 'schedule' => $schedule->getAttribute('schedule'), - 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'project' => $project, - 'function' => $function, - ]; - }; - - $schedules = []; // Local copy of 'schedules' collection - $lastSyncUpdate = DateTime::now(); - - $limit = 10000; - $sum = $limit; - $total = 0; - $loadStart = \microtime(true); - $latestDocument = null; - - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [App::getEnv('_APP_REGION')]), - Query::equal('resourceType', ['function']), - Query::equal('active', [true]), - ])); - - $sum = count($results); - $total = $total + $sum; - foreach ($results as $document) { - $schedules[$document['resourceId']] = $getSchedule($document); - } - - $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; + public static function getName(): string + { + return 'schedule'; } - Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); + public function __construct() + { + $this + ->desc('Execute functions scheduled in Appwrite') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + } - Console::success("Starting timers at " . DateTime::now()); - - Co\run( - function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { - /** - * The timer synchronize $schedules copy with database collection. - */ - Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { - $time = DateTime::now(); - $timerStart = \microtime(true); - - $limit = 1000; - $sum = $limit; - $total = 0; - $latestDocument = null; - - Console::log("Sync tick: Running at $time"); - - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [App::getEnv('_APP_REGION')]), - Query::equal('resourceType', ['function']), - Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), - ])); - - $sum = count($results); - $total = $total + $sum; - foreach ($results as $document) { - $localDocument = $schedules[$document['resourceId']] ?? null; - - $org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null; - $new = strtotime($document['resourceUpdatedAt']); - - if ($document['active'] === false) { - Console::info("Removing: {$document['resourceId']}"); - unset($schedules[$document['resourceId']]); - } elseif ($new !== $org) { - Console::info("Updating: {$document['resourceId']}"); - $schedules[$document['resourceId']] = $getSchedule($document); - } - } - $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; - } - - $lastSyncUpdate = $time; - $timerEnd = \microtime(true); - - Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); - }); - - /** - * The timer to prepare soon-to-execute schedules. - */ - $lastEnqueueUpdate = null; - $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { - $timerStart = \microtime(true); - $time = DateTime::now(); - - $enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate; - $timeFrame = DateTime::addSeconds(new \DateTime(), FUNCTION_ENQUEUE_TIMER - $enqueueDiff); - - Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)"); - - $total = 0; - - $delayedExecutions = []; // Group executions with same delay to share one coroutine - - foreach ($schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); - $nextDate = $cron->getNextRunDate(); - $next = DateTime::format($nextDate); - - $currentTick = $next < $timeFrame; - - if(!$currentTick) { - continue; - } - - $total++; - - $promiseStart = \microtime(true); // in seconds - $executionStart = $nextDate->getTimestamp(); // in seconds - $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued - - $delay = \intval($executionSleep); - - if(!isset($delayedExecutions[$delay])) { - $delayedExecutions[$delay] = []; - } - - $delayedExecutions[$delay][] = $key; - } - - foreach($delayedExecutions as $delay => $scheduleKeys) { - \go(function() use ($delay, $schedules, $scheduleKeys) { - \sleep($delay); // in seconds - - foreach($scheduleKeys as $scheduleKey) { - // Ensure schedule was not deleted - if(!isset($schedules[$scheduleKey])) { - return; - } - - Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue - } - }); - } - - $timerEnd = \microtime(true); - $lastEnqueueUpdate = $timerStart; - Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); - }; - - Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); - $enqueueFunctions(); + /** + * 1. Load all documents from 'schedules' collection to create local copy + * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute + * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker. + */ + public function action(Database $dbForConsole, callable $getProjectDB): void + { + Console::title('Scheduler V1'); + Console::success(APP_NAME . ' Scheduler v1 has started'); + + /** + * Extract only nessessary attributes to lower memory used. + * + * @var Document $schedule + * @return array + */ + $getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array { + $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId')); + $function = $getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')); + + return [ + 'resourceId' => $schedule->getAttribute('resourceId'), + 'schedule' => $schedule->getAttribute('schedule'), + 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), + 'project' => $project, + 'function' => $function, + ]; + }; + + $schedules = []; // Local copy of 'schedules' collection + $lastSyncUpdate = DateTime::now(); + + $limit = 10000; + $sum = $limit; + $total = 0; + $loadStart = \microtime(true); + $latestDocument = null; + + while ($sum === $limit) { + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ + Query::equal('region', [App::getEnv('_APP_REGION')]), + Query::equal('resourceType', ['function']), + Query::equal('active', [true]), + ])); + + $sum = count($results); + $total = $total + $sum; + foreach ($results as $document) { + $schedules[$document['resourceId']] = $getSchedule($document); + } + + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } - ); -}); + + Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); + + Console::success("Starting timers at " . DateTime::now()); + + Co\run( + function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { + /** + * The timer synchronize $schedules copy with database collection. + */ + Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { + $time = DateTime::now(); + $timerStart = \microtime(true); + + $limit = 1000; + $sum = $limit; + $total = 0; + $latestDocument = null; + + Console::log("Sync tick: Running at $time"); + + while ($sum === $limit) { + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ + Query::equal('region', [App::getEnv('_APP_REGION')]), + Query::equal('resourceType', ['function']), + Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), + ])); + + $sum = count($results); + $total = $total + $sum; + foreach ($results as $document) { + $localDocument = $schedules[$document['resourceId']] ?? null; + + $org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null; + $new = strtotime($document['resourceUpdatedAt']); + + if ($document['active'] === false) { + Console::info("Removing: {$document['resourceId']}"); + unset($schedules[$document['resourceId']]); + } elseif ($new !== $org) { + Console::info("Updating: {$document['resourceId']}"); + $schedules[$document['resourceId']] = $getSchedule($document); + } + } + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; + } + + $lastSyncUpdate = $time; + $timerEnd = \microtime(true); + + Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); + }); + + /** + * The timer to prepare soon-to-execute schedules. + */ + $lastEnqueueUpdate = null; + $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { + $timerStart = \microtime(true); + $time = DateTime::now(); + + $enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate; + $timeFrame = DateTime::addSeconds(new \DateTime(), self::FUNCTION_ENQUEUE_TIMER - $enqueueDiff); + + Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)"); + + $total = 0; + + $delayedExecutions = []; // Group executions with same delay to share one coroutine + + foreach ($schedules as $key => $schedule) { + $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); + $next = DateTime::format($nextDate); + + $currentTick = $next < $timeFrame; + + if(!$currentTick) { + continue; + } + + $total++; + + $promiseStart = \microtime(true); // in seconds + $executionStart = $nextDate->getTimestamp(); // in seconds + $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued + + $delay = \intval($executionSleep); + + if(!isset($delayedExecutions[$delay])) { + $delayedExecutions[$delay] = []; + } + + $delayedExecutions[$delay][] = $key; + } + + foreach($delayedExecutions as $delay => $scheduleKeys) { + \go(function() use ($delay, $schedules, $scheduleKeys) { + \sleep($delay); // in seconds + + foreach($scheduleKeys as $scheduleKey) { + // Ensure schedule was not deleted + if(!isset($schedules[$scheduleKey])) { + return; + } + + Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue + } + }); + } + + $timerEnd = \microtime(true); + $lastEnqueueUpdate = $timerStart; + Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); + }; + + Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); + $enqueueFunctions(); + } + ); + } +} From b031e13997005c437004b1ade7e65fe0df006fcc Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 11:39:24 +0100 Subject: [PATCH 24/65] Improve reclaim of pools in schedule --- app/cli.php | 10 +++++----- src/Appwrite/Platform/Tasks/schedule.php | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index b176326191..3546674a13 100644 --- a/app/cli.php +++ b/app/cli.php @@ -61,16 +61,16 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $dbForConsole; } - $dbAdapter = $pools + $connection = $pools ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; + ->pop(); + + $dbAdapter = $connection->getResource(); $database = new Database($dbAdapter, $cache); $database->setNamespace('_' . $project->getInternalId()); - return $database; + return [ $database, fn() => $connection->claim() ]; }; return $getProjectDB; diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php index e070206405..0d88de1d05 100644 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ b/src/Appwrite/Platform/Tasks/schedule.php @@ -49,7 +49,10 @@ class Schedule extends Action */ $getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array { $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId')); - $function = $getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')); + + [ $database, $reclaim ] = $getProjectDB($project); + $function = $database->getDocument('functions', $schedule->getAttribute('resourceId')); + $reclaim(); return [ 'resourceId' => $schedule->getAttribute('resourceId'), From c13589c1ea801d5e2ee92b2eb9d7421a14567d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 15 Nov 2022 11:04:13 +0000 Subject: [PATCH 25/65] Bug fix --- src/Appwrite/Platform/Tasks/schedule.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php index 0d88de1d05..ca1de4164e 100644 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ b/src/Appwrite/Platform/Tasks/schedule.php @@ -12,6 +12,8 @@ use Utopia\Database\Query; use Swoole\Timer; use Utopia\Database\Database; +use function Swoole\Coroutine\run; + class Schedule extends Action { const FUNCTION_UPDATE_TIMER = 10; //seconds @@ -96,7 +98,7 @@ class Schedule extends Action Console::success("Starting timers at " . DateTime::now()); - Co\run( + run( function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { /** * The timer synchronize $schedules copy with database collection. From 1084631d0f8e5b31aef5381773779c8a986d5aff Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 12:23:08 +0100 Subject: [PATCH 26/65] Fux reclaim logic in CLI; Prevent early executions --- app/cli.php | 2 +- src/Appwrite/Platform/Tasks/schedule.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/cli.php b/app/cli.php index 3546674a13..c84adef11c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -70,7 +70,7 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database = new Database($dbAdapter, $cache); $database->setNamespace('_' . $project->getInternalId()); - return [ $database, fn() => $connection->claim() ]; + return [ $database, fn() => $connection->reclaim() ]; }; return $getProjectDB; diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php index ca1de4164e..798353ef57 100644 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ b/src/Appwrite/Platform/Tasks/schedule.php @@ -184,7 +184,7 @@ class Schedule extends Action $executionStart = $nextDate->getTimestamp(); // in seconds $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued - $delay = \intval($executionSleep); + $delay = \ceil(\intval($executionSleep)); if(!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; From 9539e2790783a4a8ec26bb25a205a83bbba04edc Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 12:32:36 +0100 Subject: [PATCH 27/65] Linter fix --- src/Appwrite/Platform/Tasks/schedule.php | 90 ++++++++++++------------ 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php index 798353ef57..f9106e47c8 100644 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ b/src/Appwrite/Platform/Tasks/schedule.php @@ -16,8 +16,8 @@ use function Swoole\Coroutine\run; class Schedule extends Action { - const FUNCTION_UPDATE_TIMER = 10; //seconds - const FUNCTION_ENQUEUE_TIMER = 60; //seconds + public const FUNCTION_UPDATE_TIMER = 10; //seconds + public const FUNCTION_ENQUEUE_TIMER = 60; //seconds public static function getName(): string { @@ -42,10 +42,10 @@ class Schedule extends Action { Console::title('Scheduler V1'); Console::success(APP_NAME . ' Scheduler v1 has started'); - + /** * Extract only nessessary attributes to lower memory used. - * + * * @var Document $schedule * @return array */ @@ -55,7 +55,7 @@ class Schedule extends Action [ $database, $reclaim ] = $getProjectDB($project); $function = $database->getDocument('functions', $schedule->getAttribute('resourceId')); $reclaim(); - + return [ 'resourceId' => $schedule->getAttribute('resourceId'), 'schedule' => $schedule->getAttribute('schedule'), @@ -64,16 +64,16 @@ class Schedule extends Action 'function' => $function, ]; }; - + $schedules = []; // Local copy of 'schedules' collection $lastSyncUpdate = DateTime::now(); - + $limit = 10000; $sum = $limit; $total = 0; $loadStart = \microtime(true); $latestDocument = null; - + while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; if ($latestDocument !== null) { @@ -84,20 +84,20 @@ class Schedule extends Action Query::equal('resourceType', ['function']), Query::equal('active', [true]), ])); - + $sum = count($results); $total = $total + $sum; foreach ($results as $document) { $schedules[$document['resourceId']] = $getSchedule($document); } - + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } - + Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); - + Console::success("Starting timers at " . DateTime::now()); - + run( function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { /** @@ -106,14 +106,14 @@ class Schedule extends Action Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { $time = DateTime::now(); $timerStart = \microtime(true); - + $limit = 1000; $sum = $limit; $total = 0; $latestDocument = null; - + Console::log("Sync tick: Running at $time"); - + while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; if ($latestDocument !== null) { @@ -124,15 +124,15 @@ class Schedule extends Action Query::equal('resourceType', ['function']), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); - + $sum = count($results); $total = $total + $sum; foreach ($results as $document) { $localDocument = $schedules[$document['resourceId']] ?? null; - + $org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null; $new = strtotime($document['resourceUpdatedAt']); - + if ($document['active'] === false) { Console::info("Removing: {$document['resourceId']}"); unset($schedules[$document['resourceId']]); @@ -143,13 +143,13 @@ class Schedule extends Action } $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } - + $lastSyncUpdate = $time; $timerEnd = \microtime(true); - + Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); }); - + /** * The timer to prepare soon-to-execute schedules. */ @@ -157,62 +157,62 @@ class Schedule extends Action $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { $timerStart = \microtime(true); $time = DateTime::now(); - + $enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate; $timeFrame = DateTime::addSeconds(new \DateTime(), self::FUNCTION_ENQUEUE_TIMER - $enqueueDiff); - + Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)"); - + $total = 0; - + $delayedExecutions = []; // Group executions with same delay to share one coroutine - + foreach ($schedules as $key => $schedule) { $cron = new CronExpression($schedule['schedule']); $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); - + $currentTick = $next < $timeFrame; - - if(!$currentTick) { + + if (!$currentTick) { continue; } - + $total++; - + $promiseStart = \microtime(true); // in seconds $executionStart = $nextDate->getTimestamp(); // in seconds $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued - + $delay = \ceil(\intval($executionSleep)); - - if(!isset($delayedExecutions[$delay])) { + + if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; } - + $delayedExecutions[$delay][] = $key; } - - foreach($delayedExecutions as $delay => $scheduleKeys) { - \go(function() use ($delay, $schedules, $scheduleKeys) { + + foreach ($delayedExecutions as $delay => $scheduleKeys) { + \go(function () use ($delay, $schedules, $scheduleKeys) { \sleep($delay); // in seconds - - foreach($scheduleKeys as $scheduleKey) { + + foreach ($scheduleKeys as $scheduleKey) { // Ensure schedule was not deleted - if(!isset($schedules[$scheduleKey])) { + if (!isset($schedules[$scheduleKey])) { return; } - + Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue } }); } - + $timerEnd = \microtime(true); $lastEnqueueUpdate = $timerStart; Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); }; - + Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); $enqueueFunctions(); } From 727338cb9ac8a375cd5434886395ec8369c0ef66 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 13:54:54 +0100 Subject: [PATCH 28/65] Imrpove pools relciam logic --- app/cli.php | 9 ++++----- src/Appwrite/Platform/Tasks/schedule.php | 18 +++++++++++------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/app/cli.php b/app/cli.php index c84adef11c..0160738f68 100644 --- a/app/cli.php +++ b/app/cli.php @@ -61,16 +61,15 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $dbForConsole; } - $connection = $pools + $dbAdapter = $pools ->get($project->getAttribute('database')) - ->pop(); - - $dbAdapter = $connection->getResource(); + ->pop() + ->getResource(); $database = new Database($dbAdapter, $cache); $database->setNamespace('_' . $project->getInternalId()); - return [ $database, fn() => $connection->reclaim() ]; + return $database; }; return $getProjectDB; diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php index f9106e47c8..36c28be141 100644 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ b/src/Appwrite/Platform/Tasks/schedule.php @@ -11,6 +11,7 @@ use Utopia\Database\Document; use Utopia\Database\Query; use Swoole\Timer; use Utopia\Database\Database; +use Utopia\Pools\Group; use function Swoole\Coroutine\run; @@ -28,9 +29,10 @@ class Schedule extends Action { $this ->desc('Execute functions scheduled in Appwrite') + ->inject('pools') ->inject('dbForConsole') ->inject('getProjectDB') - ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + ->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB)); } /** @@ -38,7 +40,7 @@ class Schedule extends Action * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker. */ - public function action(Database $dbForConsole, callable $getProjectDB): void + public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void { Console::title('Scheduler V1'); Console::success(APP_NAME . ' Scheduler v1 has started'); @@ -52,9 +54,7 @@ class Schedule extends Action $getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array { $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId')); - [ $database, $reclaim ] = $getProjectDB($project); - $function = $database->getDocument('functions', $schedule->getAttribute('resourceId')); - $reclaim(); + $function = $getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')); return [ 'resourceId' => $schedule->getAttribute('resourceId'), @@ -93,17 +93,19 @@ class Schedule extends Action $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } + + $pools->reclaim(); Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); Console::success("Starting timers at " . DateTime::now()); run( - function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { + function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) { /** * The timer synchronize $schedules copy with database collection. */ - Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule) { + Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) { $time = DateTime::now(); $timerStart = \microtime(true); @@ -147,6 +149,8 @@ class Schedule extends Action $lastSyncUpdate = $time; $timerEnd = \microtime(true); + $pools->reclaim(); + Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); }); From ea5bd5160a2fda5339204328c5a1ed8b4fd5bea6 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 14:01:06 +0100 Subject: [PATCH 29/65] Linter fix --- src/Appwrite/Platform/Tasks/schedule.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php index 36c28be141..2598f9cd62 100644 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ b/src/Appwrite/Platform/Tasks/schedule.php @@ -77,7 +77,7 @@ class Schedule extends Action while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); + $paginationQueries[] = Query::cursorAfter($latestDocument); } $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ Query::equal('region', [App::getEnv('_APP_REGION')]), @@ -93,7 +93,7 @@ class Schedule extends Action $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } - + $pools->reclaim(); Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); From bb3c99c58bd7d806a7dd645e852605420a4f8cc2 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 14:01:24 +0100 Subject: [PATCH 30/65] Rename file step 1 --- src/Appwrite/Platform/Tasks/schedule.php | 225 ----------------------- 1 file changed, 225 deletions(-) delete mode 100644 src/Appwrite/Platform/Tasks/schedule.php diff --git a/src/Appwrite/Platform/Tasks/schedule.php b/src/Appwrite/Platform/Tasks/schedule.php deleted file mode 100644 index 2598f9cd62..0000000000 --- a/src/Appwrite/Platform/Tasks/schedule.php +++ /dev/null @@ -1,225 +0,0 @@ -desc('Execute functions scheduled in Appwrite') - ->inject('pools') - ->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB)); - } - - /** - * 1. Load all documents from 'schedules' collection to create local copy - * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute - * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker. - */ - public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void - { - Console::title('Scheduler V1'); - Console::success(APP_NAME . ' Scheduler v1 has started'); - - /** - * Extract only nessessary attributes to lower memory used. - * - * @var Document $schedule - * @return array - */ - $getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array { - $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId')); - - $function = $getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')); - - return [ - 'resourceId' => $schedule->getAttribute('resourceId'), - 'schedule' => $schedule->getAttribute('schedule'), - 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'project' => $project, - 'function' => $function, - ]; - }; - - $schedules = []; // Local copy of 'schedules' collection - $lastSyncUpdate = DateTime::now(); - - $limit = 10000; - $sum = $limit; - $total = 0; - $loadStart = \microtime(true); - $latestDocument = null; - - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [App::getEnv('_APP_REGION')]), - Query::equal('resourceType', ['function']), - Query::equal('active', [true]), - ])); - - $sum = count($results); - $total = $total + $sum; - foreach ($results as $document) { - $schedules[$document['resourceId']] = $getSchedule($document); - } - - $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; - } - - $pools->reclaim(); - - Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); - - Console::success("Starting timers at " . DateTime::now()); - - run( - function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) { - /** - * The timer synchronize $schedules copy with database collection. - */ - Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) { - $time = DateTime::now(); - $timerStart = \microtime(true); - - $limit = 1000; - $sum = $limit; - $total = 0; - $latestDocument = null; - - Console::log("Sync tick: Running at $time"); - - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [App::getEnv('_APP_REGION')]), - Query::equal('resourceType', ['function']), - Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), - ])); - - $sum = count($results); - $total = $total + $sum; - foreach ($results as $document) { - $localDocument = $schedules[$document['resourceId']] ?? null; - - $org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null; - $new = strtotime($document['resourceUpdatedAt']); - - if ($document['active'] === false) { - Console::info("Removing: {$document['resourceId']}"); - unset($schedules[$document['resourceId']]); - } elseif ($new !== $org) { - Console::info("Updating: {$document['resourceId']}"); - $schedules[$document['resourceId']] = $getSchedule($document); - } - } - $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; - } - - $lastSyncUpdate = $time; - $timerEnd = \microtime(true); - - $pools->reclaim(); - - Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); - }); - - /** - * The timer to prepare soon-to-execute schedules. - */ - $lastEnqueueUpdate = null; - $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { - $timerStart = \microtime(true); - $time = DateTime::now(); - - $enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate; - $timeFrame = DateTime::addSeconds(new \DateTime(), self::FUNCTION_ENQUEUE_TIMER - $enqueueDiff); - - Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)"); - - $total = 0; - - $delayedExecutions = []; // Group executions with same delay to share one coroutine - - foreach ($schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); - $nextDate = $cron->getNextRunDate(); - $next = DateTime::format($nextDate); - - $currentTick = $next < $timeFrame; - - if (!$currentTick) { - continue; - } - - $total++; - - $promiseStart = \microtime(true); // in seconds - $executionStart = $nextDate->getTimestamp(); // in seconds - $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued - - $delay = \ceil(\intval($executionSleep)); - - if (!isset($delayedExecutions[$delay])) { - $delayedExecutions[$delay] = []; - } - - $delayedExecutions[$delay][] = $key; - } - - foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $schedules, $scheduleKeys) { - \sleep($delay); // in seconds - - foreach ($scheduleKeys as $scheduleKey) { - // Ensure schedule was not deleted - if (!isset($schedules[$scheduleKey])) { - return; - } - - Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue - } - }); - } - - $timerEnd = \microtime(true); - $lastEnqueueUpdate = $timerStart; - Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); - }; - - Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); - $enqueueFunctions(); - } - ); - } -} From 451c6334bb2acc664913498c3c83cee357b67ec6 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 14:01:44 +0100 Subject: [PATCH 31/65] Rename file step 2 --- src/Appwrite/Platform/Tasks/Schedule.php | 225 +++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 src/Appwrite/Platform/Tasks/Schedule.php diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php new file mode 100644 index 0000000000..2598f9cd62 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -0,0 +1,225 @@ +desc('Execute functions scheduled in Appwrite') + ->inject('pools') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB)); + } + + /** + * 1. Load all documents from 'schedules' collection to create local copy + * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute + * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker. + */ + public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void + { + Console::title('Scheduler V1'); + Console::success(APP_NAME . ' Scheduler v1 has started'); + + /** + * Extract only nessessary attributes to lower memory used. + * + * @var Document $schedule + * @return array + */ + $getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array { + $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId')); + + $function = $getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')); + + return [ + 'resourceId' => $schedule->getAttribute('resourceId'), + 'schedule' => $schedule->getAttribute('schedule'), + 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), + 'project' => $project, + 'function' => $function, + ]; + }; + + $schedules = []; // Local copy of 'schedules' collection + $lastSyncUpdate = DateTime::now(); + + $limit = 10000; + $sum = $limit; + $total = 0; + $loadStart = \microtime(true); + $latestDocument = null; + + while ($sum === $limit) { + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ + Query::equal('region', [App::getEnv('_APP_REGION')]), + Query::equal('resourceType', ['function']), + Query::equal('active', [true]), + ])); + + $sum = count($results); + $total = $total + $sum; + foreach ($results as $document) { + $schedules[$document['resourceId']] = $getSchedule($document); + } + + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; + } + + $pools->reclaim(); + + Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); + + Console::success("Starting timers at " . DateTime::now()); + + run( + function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) { + /** + * The timer synchronize $schedules copy with database collection. + */ + Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) { + $time = DateTime::now(); + $timerStart = \microtime(true); + + $limit = 1000; + $sum = $limit; + $total = 0; + $latestDocument = null; + + Console::log("Sync tick: Running at $time"); + + while ($sum === $limit) { + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ + Query::equal('region', [App::getEnv('_APP_REGION')]), + Query::equal('resourceType', ['function']), + Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), + ])); + + $sum = count($results); + $total = $total + $sum; + foreach ($results as $document) { + $localDocument = $schedules[$document['resourceId']] ?? null; + + $org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null; + $new = strtotime($document['resourceUpdatedAt']); + + if ($document['active'] === false) { + Console::info("Removing: {$document['resourceId']}"); + unset($schedules[$document['resourceId']]); + } elseif ($new !== $org) { + Console::info("Updating: {$document['resourceId']}"); + $schedules[$document['resourceId']] = $getSchedule($document); + } + } + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; + } + + $lastSyncUpdate = $time; + $timerEnd = \microtime(true); + + $pools->reclaim(); + + Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); + }); + + /** + * The timer to prepare soon-to-execute schedules. + */ + $lastEnqueueUpdate = null; + $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { + $timerStart = \microtime(true); + $time = DateTime::now(); + + $enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate; + $timeFrame = DateTime::addSeconds(new \DateTime(), self::FUNCTION_ENQUEUE_TIMER - $enqueueDiff); + + Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)"); + + $total = 0; + + $delayedExecutions = []; // Group executions with same delay to share one coroutine + + foreach ($schedules as $key => $schedule) { + $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); + $next = DateTime::format($nextDate); + + $currentTick = $next < $timeFrame; + + if (!$currentTick) { + continue; + } + + $total++; + + $promiseStart = \microtime(true); // in seconds + $executionStart = $nextDate->getTimestamp(); // in seconds + $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued + + $delay = \ceil(\intval($executionSleep)); + + if (!isset($delayedExecutions[$delay])) { + $delayedExecutions[$delay] = []; + } + + $delayedExecutions[$delay][] = $key; + } + + foreach ($delayedExecutions as $delay => $scheduleKeys) { + \go(function () use ($delay, $schedules, $scheduleKeys) { + \sleep($delay); // in seconds + + foreach ($scheduleKeys as $scheduleKey) { + // Ensure schedule was not deleted + if (!isset($schedules[$scheduleKey])) { + return; + } + + Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue + } + }); + } + + $timerEnd = \microtime(true); + $lastEnqueueUpdate = $timerStart; + Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); + }; + + Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); + $enqueueFunctions(); + } + ); + } +} From bfe8b9800862a1526bbb217348db19dbe0a9e7cd Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 14:30:19 +0100 Subject: [PATCH 32/65] Attempt to reuse db connections --- app/cli.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/cli.php b/app/cli.php index 0160738f68..9c77278c7e 100644 --- a/app/cli.php +++ b/app/cli.php @@ -56,19 +56,29 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { }, ['pools', 'cache']); CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { - $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache) { + $databases = []; + + $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } + $databaseName = $project->getAttribute('database'); + + if(isset($databases[$databaseName])) { + return $databases[$databaseName]; + } + $dbAdapter = $pools - ->get($project->getAttribute('database')) + ->get($databaseName) ->pop() ->getResource(); $database = new Database($dbAdapter, $cache); $database->setNamespace('_' . $project->getInternalId()); + $databases[$databaseName] = $database; + return $database; }; From b6e4ec822bfb554bb8e910a3de35e4018610a102 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 14:34:13 +0100 Subject: [PATCH 33/65] Add TODO --- app/cli.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/cli.php b/app/cli.php index 9c77278c7e..04e4d9e4ff 100644 --- a/app/cli.php +++ b/app/cli.php @@ -56,7 +56,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { }, ['pools', 'cache']); CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { - $databases = []; + $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { From c07953a640ac2aaa7f0fe12eb0440ed936f93366 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 14:35:13 +0100 Subject: [PATCH 34/65] Linter fix --- app/cli.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/cli.php b/app/cli.php index 04e4d9e4ff..502ee77b75 100644 --- a/app/cli.php +++ b/app/cli.php @@ -65,7 +65,7 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databaseName = $project->getAttribute('database'); - if(isset($databases[$databaseName])) { + if (isset($databases[$databaseName])) { return $databases[$databaseName]; } From e04295918f27a7be337cf787aab0f7d969d7f880 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 14:36:43 +0100 Subject: [PATCH 35/65] Add TODO --- src/Appwrite/Platform/Tasks/Schedule.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index 2598f9cd62..0e48d23883 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -60,8 +60,8 @@ class Schedule extends Action 'resourceId' => $schedule->getAttribute('resourceId'), 'schedule' => $schedule->getAttribute('schedule'), 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'project' => $project, - 'function' => $function, + 'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here + 'function' => $function, // TODO: @Meldiron Send only ID to worker to reduce memory usage here ]; }; From 390fbc52ebbd1852763a32dedfcf233636552e0a Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 16:33:43 +0200 Subject: [PATCH 36/65] minor changes --- .env | 2 +- app/controllers/api/functions.php | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.env b/.env index 53aa2ac7c2..390e838d8f 100644 --- a/.env +++ b/.env @@ -87,6 +87,6 @@ _APP_USAGE_DATABASE_INTERVAL=15 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= -_APP_REGION=nyc1 +_APP_REGION=default _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 2549a6b64b..4e844f95a2 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -547,9 +547,7 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $schedule) { - $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); - }); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $events ->setParam('functionId', $function->getId()) @@ -597,9 +595,7 @@ App::delete('/v1/functions/:functionId') ->setAttribute('active', false) ; - Authorization::skip(function () use ($dbForConsole, $schedule) { - $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); - }); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $deletes ->setType(DELETE_TYPE_DOCUMENT) @@ -804,9 +800,7 @@ App::post('/v1/functions/:functionId/deployments') $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $schedule) { - $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); - }); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $metadata = null; From 8bcf349c3814c996abebd12740c2ffe45a71cd5a Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:03:42 +0200 Subject: [PATCH 37/65] addressing some comments --- app/controllers/api/functions.php | 10 +++++----- app/init.php | 17 +++++------------ app/worker.php | 4 +--- app/workers/functions.php | 21 +++++++++++---------- 4 files changed, 22 insertions(+), 30 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5b969e63d8..6f09e27385 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -41,7 +41,7 @@ use Utopia\CLI\Console; use Utopia\Database\Validator\Roles; use Utopia\Validator\Boolean; use Utopia\Database\Exception\Duplicate as DuplicateException; -use Utopia\Queue\Client as queue; +use Utopia\Queue\Client as QueueClient; include_once __DIR__ . '/../shared/api.php'; @@ -1155,10 +1155,10 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $queue = new queue(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); - $queue->enqueue([ - 'type' => 'http', - 'value' => [ + $queueForFunctions = new QueueClient(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); + $queueForFunctions->enqueue([ + 'type' => 'http', + 'value' => [ 'type' => 'http', 'execution' => $execution, 'function' => $function, diff --git a/app/init.php b/app/init.php index 4500526583..c5e898d836 100644 --- a/app/init.php +++ b/app/init.php @@ -38,8 +38,6 @@ use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; use Appwrite\URL\URL as AppwriteURL; -use Utopia\Queue\Client as SyncOut; -use Utopia\Queue\Connection\Redis as QueueRedis; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; @@ -67,7 +65,6 @@ use Utopia\Storage\Device\Wasabi; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; -use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\MySQL; use Utopia\Pools\Group; @@ -526,35 +523,30 @@ $register->set('pools', function () { 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], - 'useResource' => true, ], 'database' => [ 'type' => 'database', 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], - 'useResource' => true, ], 'queue' => [ 'type' => 'queue', 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], - 'useResource' => false, ], 'pubsub' => [ 'type' => 'pubsub', 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], - 'useResource' => true, ], 'cache' => [ 'type' => 'cache', 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), 'multiple' => true, 'schemes' => ['redis'], - 'useResource' => true, ], ]; @@ -586,7 +578,7 @@ $register->set('pools', function () { $dsnScheme = $dsn->getScheme(); $dsnDatabase = $dsn->getDatabase(); - if (!in_array($dsnScheme, $schemes) && $useResource) { + if (!in_array($dsnScheme, $schemes)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); } @@ -647,12 +639,12 @@ $register->set('pools', function () { $adapter->setDefaultDatabase($dsn->getDatabase()); break; case 'pubsub': - break; $adapter = $resource(); + break; case 'queue': $adapter = match ($dsn->getScheme()) { 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), - default => 'bla' + default => null }; break; case 'cache': @@ -678,6 +670,7 @@ $register->set('pools', function () { return $group; }); + $register->set('influxdb', function () { // Register DB connection $host = App::getEnv('_APP_INFLUXDB_HOST', ''); @@ -853,7 +846,7 @@ App::setResource('messaging', fn() => new Phone()); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); -App::setResource('clients', function ($request, $console, $project) use ($register) { +App::setResource('clients', function ($request, $console, $project) { $console->setAttribute('platforms', [ // Always allow current host '$collection' => ID::custom('platforms'), 'name' => 'Current Host', diff --git a/app/worker.php b/app/worker.php index bb35576172..4e484e1545 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,7 +2,6 @@ require_once __DIR__ . '/init.php'; - use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -14,7 +13,6 @@ use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; - global $register; Server::setResource('register', fn() => $register); @@ -76,7 +74,7 @@ App::setResource('logger', function ($register) { $pools = $register->get('pools'); -$client = $pools->get('queue')->pop()->getResource(); +$connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); diff --git a/app/workers/functions.php b/app/workers/functions.php index 86fc2998b3..91edee19fe 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -20,11 +20,12 @@ use Utopia\Database\Query; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; +use Utopia\Queue\Client as QueueClient; Authorization::disable(); Authorization::setDefaultStatus(false); -global $client; +global $connection; global $workerNumber; $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -40,7 +41,7 @@ $execute = function ( string $data = null, ?Document $user = null, string $jwt = null -) use ($executor) { +) use ($executor, $register) { $user ??= new Document(); $functionId = $function->getId(); @@ -50,28 +51,28 @@ $execute = function ( $deployment = $dbForProject->getDocument('deployments', $deploymentId); if ($deployment->getAttribute('resourceId') !== $functionId) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } if ($deployment->isEmpty()) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } /** Check if build has exists */ $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($build->isEmpty()) { - throw new Exception('Build not found', 404); + throw new Exception('Build not found'); } if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready', 400); + throw new Exception('Build not ready'); } /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } $runtime = $runtimes[$function->getAttribute('runtime')]; @@ -102,14 +103,14 @@ $execute = function ( $execution = $dbForProject->updateDocument('executions', $executionId, $execution); if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready', 400); + throw new Exception('Build not ready'); } /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } $runtime = $runtimes[$function->getAttribute('runtime')]; @@ -254,7 +255,7 @@ $execute = function ( } }; -$adapter = new Queue\Adapter\Swoole($client, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); +$adapter = new Queue\Adapter\Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); $server = new Queue\Server($adapter); $server->job() From 3682c1d05bbfe6a5c18355655936aa3da53c30ae Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:07:13 +0200 Subject: [PATCH 38/65] addressing some comments --- bin/worker-functions | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/bin/worker-functions b/bin/worker-functions index 5e9728c46e..7e7b4e565f 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=0.1 QUEUE='v1-functions' APP_INCLUDE='/usr/src/code/app/workers/functions.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/cli.php worker-functions $@ \ No newline at end of file From 0d79f19dcd2c4f4f22e1d5e8336ff9965fabecc4 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:14:52 +0200 Subject: [PATCH 39/65] addressing some comments --- app/controllers/api/functions.php | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 6f09e27385..fb688aaae3 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -90,7 +90,7 @@ App::post('/v1/functions') 'search' => implode(' ', [$functionId, $name, $runtime]) ])); - $log = Authorization::skip( + $schedule = Authorization::skip( fn() => $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region 'resourceType' => 'function', @@ -102,7 +102,7 @@ App::post('/v1/functions') ])) ); - $function->setAttribute('scheduleId', $log->getId()); + $function->setAttribute('scheduleId', $schedule->getId()); $dbForProject->updateDocument('functions', $function->getId(), $function); $eventsInstance->setParam('functionId', $function->getId()); @@ -470,21 +470,21 @@ App::put('/v1/functions/:functionId') 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); /** * In case we want to clear the schedule */ if (!empty($function->getAttribute('deployment'))) { - $log->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); + $schedule->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); } - $log + $schedule ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); $eventsInstance->setParam('functionId', $function->getId()); @@ -538,18 +538,18 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') 'deployment' => $deployment->getId() ]))); - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); $active = !empty($function->getAttribute('schedule')); if ($active) { - $log->setAttribute('resourceUpdatedAt', datetime::now()); + $schedule->setAttribute('resourceUpdatedAt', datetime::now()); } - $log->setAttribute('active', $active); + $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $events @@ -591,15 +591,15 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); - $log + $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('active', false) ; - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $deletes @@ -795,18 +795,18 @@ App::post('/v1/functions/:functionId/deployments') * TODO Should we update also the function collection with the scheduleUpdatedAt attr? */ - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); $active = !empty($function->getAttribute('schedule')); if ($active) { - $log->setAttribute('resourceUpdatedAt', datetime::now()); + $schedule->setAttribute('resourceUpdatedAt', datetime::now()); } - $log->setAttribute('active', $active); + $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $metadata = null; From 11915e1667bafbd36020e3c6bd6e18323f8be17f Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:20:50 +0200 Subject: [PATCH 40/65] addressing some comments --- bin/worker-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/worker-functions b/bin/worker-functions index 7e7b4e565f..6eaaeafe00 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php worker-functions $@ \ No newline at end of file +php php /usr/src/code/app/workers/worker-functions.php $@ \ No newline at end of file From 797c3f2443a020c6d836b14fc78a812562ac6d23 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:21:54 +0200 Subject: [PATCH 41/65] addressing some comments --- bin/worker-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/worker-functions b/bin/worker-functions index 6eaaeafe00..143b431b0c 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php php /usr/src/code/app/workers/worker-functions.php $@ \ No newline at end of file +php php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file From 855c3d3b84f48621470081def0e3999c2595d650 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 17:37:16 +0100 Subject: [PATCH 42/65] Re-implement removed stuff during merge --- src/Appwrite/Platform/Tasks/Schedule.php | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index 0e48d23883..39e0686da5 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -12,6 +12,8 @@ use Utopia\Database\Query; use Swoole\Timer; use Utopia\Database\Database; use Utopia\Pools\Group; +use Utopia\Queue\Client as Worker; +use Appwrite\Event\Event; use function Swoole\Coroutine\run; @@ -158,7 +160,7 @@ class Schedule extends Action * The timer to prepare soon-to-execute schedules. */ $lastEnqueueUpdate = null; - $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { + $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate, $pools) { $timerStart = \microtime(true); $time = DateTime::now(); @@ -198,7 +200,7 @@ class Schedule extends Action } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $schedules, $scheduleKeys) { + \go(function () use ($delay, $schedules, $scheduleKeys, $pools) { \sleep($delay); // in seconds foreach ($scheduleKeys as $scheduleKey) { @@ -207,7 +209,21 @@ class Schedule extends Action return; } - Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue + $schedule = $schedules[$scheduleKey]; + + $queue = $pools->get('queue')->pop(); + + $worker = new Worker(Event::FUNCTIONS_QUEUE_NAME, $queue->getResource()); + $worker + ->enqueue([ + 'type' => 'schedule', + 'value' => [ + 'project' => $schedule['project'], + 'function' => $schedule['function'], + ] + ]); + + $queue->reclaim(); } }); } From 280a44e1cd2e3dd0e35d3be5b0ae0ee60c59c00f Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 17:55:30 +0100 Subject: [PATCH 43/65] Fix bugs during QA --- app/workers/deletes.php | 26 ++------------ app/workers/functions.php | 13 ++++--- bin/worker-functions | 2 +- composer.lock | 72 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 79 insertions(+), 34 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 0134fad66d..3e0df8ce38 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -468,19 +468,7 @@ class DeletesV1 extends Worker Query::equal('functionId', [$functionId]) ], $dbForProject); - /** - * Request executor to delete all deployment containers - * TODO: Re-enable. Disabled for now because of proxy. Container killed after inactivity automatically. - */ - // Console::info("Requesting executor to delete all deployment containers for function " . $functionId); - // $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - // foreach ($deploymentIds as $deploymentId) { - // try { - // $executor->deleteRuntime($projectId, $deploymentId); - // } catch (Throwable $th) { - // Console::error($th->getMessage()); - // } - // } + // TODO: Request executor to delete runtime } /** @@ -520,17 +508,7 @@ class DeletesV1 extends Worker } }); - /** - * Request executor to delete the deployment container. - * TODO: Re-enable. Disabled for now because of proxy. Container killed after inactivity automatically. - */ - // Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId); - // try { - // $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - // $executor->deleteRuntime($projectId, $deploymentId); - // } catch (Throwable $th) { - // Console::error($th->getMessage()); - // } + // TODO: Request executor to delete runtime } diff --git a/app/workers/functions.php b/app/workers/functions.php index 8a3940de8a..2d7f4565c4 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -164,14 +164,13 @@ $execute = function ( try { $executionResponse = $executor->createExecution( projectId: $project->getId(), - deploymentId: $deploymentId, - path: $build->getAttribute('outputPath', ''), - vars: $vars, + deploymentId: $deployment->getId(), + payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', + variables: $vars, + timeout: $function->getAttribute('timeout', 0), + image: $runtime['image'], + source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), - data: $vars['APPWRITE_FUNCTION_DATA'] ?? '', - runtime: $function->getAttribute('runtime', ''), - baseImage: $runtime['image'], - timeout: $function->getAttribute('timeout', 0) ); /** Update execution status */ diff --git a/bin/worker-functions b/bin/worker-functions index 143b431b0c..b22ee65d39 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file +php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file diff --git a/composer.lock b/composer.lock index f042279e4f..3912636eea 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b125039c64ae4cbe0d2a1b57322d0ebe", + "content-hash": "3e24f0f02ec826898d50e4119f9eb226", "packages": [ { "name": "adhocore/jwt", @@ -2357,6 +2357,67 @@ }, "time": "2020-10-24T07:04:59+00:00" }, + { + "name": "utopia-php/queue", + "version": "dev-upgrade-libs", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/queue.git", + "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/310aaac74d2287d3d9450a532658247cdfe5e72c", + "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.14.*", + "utopia-php/framework": "0.*.*" + }, + "require-dev": { + "laravel/pint": "^0.2.3", + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.5.5", + "swoole/ide-helper": "4.8.8", + "workerman/workerman": "^4.0" + }, + "suggest": { + "ext-swoole": "Needed to support Swoole.", + "workerman/workerman": "Needed to support Workerman." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Queue\\": "src/Queue" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A powerful task queue.", + "keywords": [ + "Tasks", + "framework", + "php", + "queue", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/queue/issues", + "source": "https://github.com/utopia-php/queue/tree/upgrade-libs" + }, + "time": "2022-11-15T16:35:56+00:00" + }, { "name": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5216,6 +5277,12 @@ } ], "aliases": [ + { + "package": "utopia-php/queue", + "version": "dev-upgrade-libs", + "alias": "0.4.1", + "alias_normalized": "0.4.1.0" + }, { "package": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5225,6 +5292,7 @@ ], "minimum-stability": "stable", "stability-flags": { + "utopia-php/queue": 20, "utopia-php/registry": 20 }, "prefer-stable": false, @@ -5250,5 +5318,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } From 44a82de09bfa23b9eda9a7dc03448d6a8a9b5424 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 19:13:17 +0100 Subject: [PATCH 44/65] Refactor func event triggering --- app/cli.php | 5 +++ app/controllers/api/functions.php | 26 +++++++-------- app/controllers/shared/api.php | 14 +++++--- app/init.php | 15 +++++---- app/worker.php | 8 ++++- app/workers/builds.php | 19 ++++++++--- app/workers/functions.php | 41 ++++++++++-------------- composer.json | 2 +- composer.lock | 21 ++++-------- src/Appwrite/Event/Func.php | 40 +++++++---------------- src/Appwrite/Platform/Tasks/Schedule.php | 24 +++++++------- 11 files changed, 106 insertions(+), 109 deletions(-) diff --git a/app/cli.php b/app/cli.php index 502ee77b75..544148c8d5 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,6 +3,7 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; +use Appwrite\Event\Func; use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; @@ -109,6 +110,10 @@ CLI::setResource('influxdb', function (Registry $register) { return $database; }, ['register']); +CLI::setResource('functions', function (Group $pools) { + return new Func($pools->get('queue')->pop()->getResource()); +}, ['pools']); + CLI::setResource('logError', function (Registry $register) { return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d9f63a93be..02c3fce885 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -5,6 +5,7 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Func; use Appwrite\Event\Validator\Event as ValidatorEvent; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\CustomId; @@ -41,7 +42,6 @@ use Utopia\CLI\Console; use Utopia\Database\Validator\Roles; use Utopia\Validator\Boolean; use Utopia\Database\Exception\Duplicate as DuplicateException; -use Utopia\Queue\Client as QueueClient; include_once __DIR__ . '/../shared/api.php'; @@ -1061,7 +1061,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('usage') ->inject('mode') ->inject('pools') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools) { + ->inject('functions') + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools, Func $functions) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1149,18 +1150,15 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $queueForFunctions = new QueueClient(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); - $queueForFunctions->enqueue([ - 'type' => 'http', - 'value' => [ - 'type' => 'http', - 'execution' => $execution, - 'function' => $function, - 'data' => $data, - 'jwt' => $jwt, - 'project' => $project, - 'user' => $user - ]]); + $functions + ->setType('http') + ->setExecution($execution) + ->setFunction($function) + ->setData($data) + ->setJWT($jwt) + ->setProject($project) + ->setUser($user) + ->trigger(); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e07f405140..58dd2f16ef 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -5,6 +5,7 @@ use Appwrite\Event\Audit; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Usage\Stats; @@ -251,7 +252,8 @@ App::shutdown() ->inject('database') ->inject('mode') ->inject('dbForProject') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject) use ($parseLabel) { + ->inject('functions') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $functions) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -262,9 +264,13 @@ App::shutdown() /** * Trigger functions. */ - $events - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + $functions + ->setData(\json_encode($events->getPayload())) + ->setProject($events->getProject()) + ->setUser($events->getUser()) + ->setEvent($events->getEvent()) + ->setParam('functionId', $events->getParam('functionId')) + ->setParam('executionId', $events->getParam('executionId')) ->trigger(); /** diff --git a/app/init.php b/app/init.php index b861297cef..19cea371df 100644 --- a/app/init.php +++ b/app/init.php @@ -71,6 +71,7 @@ use Utopia\Pools\Group; use Utopia\Pools\Pool; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; +use Appwrite\Event\Func; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; @@ -843,6 +844,9 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); +App::setResource('functions', function (Group $pools) { + return new Func($pools->get('queue')->pop()->getResource()); +}, ['pools']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); @@ -1023,13 +1027,12 @@ App::setResource('console', function () { }, []); App::setResource('queue', function () { - $fallbackForRedis = AppwriteURL::unparse([ - 'scheme' => 'redis', - 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), - 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), - 'user' => App::getEnv('_APP_REDIS_USER', ''), - 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), ]); $connection = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); diff --git a/app/worker.php b/app/worker.php index 4e484e1545..dabb91f1d8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,6 +2,7 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Event\Func; use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -68,7 +69,12 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); -App::setResource('logger', function ($register) { +Server::setResource('functions', function (Registry $register) { + $pools = $register->get('pools'); + return new Func($pools->get('queue')->pop()->getResource()); +}, ['register']); + +Server::setResource('logger', function ($register) { return $register->get('logger'); }, ['register']); diff --git a/app/workers/builds.php b/app/workers/builds.php index babf2874d6..d52329b107 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -1,6 +1,7 @@ setAttribute('status', 'building'); $build = $dbForProject->updateDocument('builds', $buildId, $build); + $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); + /** Trigger Webhook */ $deploymentModel = new Deployment(); @@ -114,14 +117,22 @@ class BuildsV1 extends Worker ->setEvent('functions.[functionId].deployments.[deploymentId].update') ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) + ->setPayload($data) ->trigger(); /** Trigger Functions */ - $deploymentUpdate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + global $register; + $pools = $register->get('pools'); + $connection = $pools->get('queue')->pop(); + $functions = new Func($connection->getResource()); + $functions + ->setData(\json_encode($data)) + ->setProject($project) + ->setEvent('functions.[functionId].deployments.[deploymentId].update') + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) ->trigger(); + $connection->reclaim(); /** Trigger Realtime */ $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ diff --git a/app/workers/functions.php b/app/workers/functions.php index 2d7f4565c4..bad6e7d640 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/../worker.php'; use Utopia\Queue; use Utopia\Queue\Message; use Appwrite\Event\Event; +use Appwrite\Event\Func; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Usage\Stats; use Appwrite\Utopia\Response\Model\Execution; @@ -33,6 +34,7 @@ $execute = function ( Document $project, Document $function, Database $dbForProject, + Func $functions, string $trigger, string $executionId = null, string $event = null, @@ -206,9 +208,13 @@ $execute = function ( ->trigger(); /** Trigger Functions */ - $executionUpdate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + $functions + ->setData($data) + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) ->trigger(); /** Trigger realtime event */ @@ -259,9 +265,10 @@ $server = new Queue\Server($adapter); $server->job() ->inject('message') ->inject('dbForProject') - ->action(function (Message $message, Database $dbForProject) use ($execute) { - $args = $message->getPayload()['value'] ?? []; - $type = $message->getPayload()['type'] ?? ''; + ->inject('functions') + ->action(function (Message $message, Database $dbForProject, Func $functions) use ($execute) { + $args = $message->getPayload() ?? []; + $type = $args['type'] ?? ''; $events = $args['events'] ?? []; $project = new Document($args['project'] ?? []); $user = new Document($args['user'] ?? []); @@ -317,34 +324,20 @@ $server->job() /** * Handle Schedule and HTTP execution. */ - $user = new Document($args['user'] ?? []); $project = new Document($args['project'] ?? []); - $execution = new Document($args['execution'] ?? []); $function = new Document($args['function'] ?? []); switch ($type) { case 'http': $jwt = $args['jwt'] ?? ''; $data = $args['data'] ?? ''; + $execution = new Document($args['execution'] ?? []); + $user = new Document($args['user'] ?? []); $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); - call_user_func($execute, $project, $function, $dbForProject, 'http', $execution->getId(), null, null, $data, $user, $jwt); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt); break; - case 'schedule': - /* - * 1. Get Original Task - * 2. Check for updates - * If has updates skip task and don't reschedule - * If status not equal to play skip task - * 3. Check next run date, update task and add new job at the given date - * 4. Execute task (set optional timeout) - * 5. Update task response to log - * On success reset error count - * On failure add error count - * If error count bigger than allowed change status to pause - */ - - call_user_func($execute, $project, $function, $dbForProject, 'schedule', null, null, null, null, null, null); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null); break; } }); diff --git a/composer.json b/composer.json index d4c06d7614..08ede65dcf 100644 --- a/composer.json +++ b/composer.json @@ -53,7 +53,7 @@ "utopia-php/domains": "1.1.*", "utopia-php/framework": "0.25.*", "utopia-php/image": "0.5.*", - "utopia-php/queue": "dev-upgrade-libs as 0.4.1", + "utopia-php/queue": "0.4.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", "utopia-php/orchestration": "0.9.*", diff --git a/composer.lock b/composer.lock index 3912636eea..674e42a5ca 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3e24f0f02ec826898d50e4119f9eb226", + "content-hash": "f2faa670abaa356f9b14e4f1c3542b33", "packages": [ { "name": "adhocore/jwt", @@ -2359,16 +2359,16 @@ }, { "name": "utopia-php/queue", - "version": "dev-upgrade-libs", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c" + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/310aaac74d2287d3d9450a532658247cdfe5e72c", - "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", "shasum": "" }, "require": { @@ -2414,9 +2414,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/upgrade-libs" + "source": "https://github.com/utopia-php/queue/tree/0.4.1" }, - "time": "2022-11-15T16:35:56+00:00" + "time": "2022-11-15T16:56:37+00:00" }, { "name": "utopia-php/registry", @@ -5277,12 +5277,6 @@ } ], "aliases": [ - { - "package": "utopia-php/queue", - "version": "dev-upgrade-libs", - "alias": "0.4.1", - "alias_normalized": "0.4.1.0" - }, { "package": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5292,7 +5286,6 @@ ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/queue": 20, "utopia-php/registry": 20 }, "prefer-stable": false, diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index b7531cf475..514eb5df74 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -6,6 +6,8 @@ use DateTime; use Resque; use ResqueScheduler; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Func extends Event { @@ -15,7 +17,7 @@ class Func extends Event protected ?Document $function = null; protected ?Document $execution = null; - public function __construct() + public function __construct(protected Connection $connection) { parent::__construct(Event::FUNCTIONS_QUEUE_NAME, Event::FUNCTIONS_CLASS_NAME); } @@ -143,36 +145,16 @@ class Func extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ - 'project' => $this->project, - 'user' => $this->user, - 'function' => $this->function, - 'execution' => $this->execution, - 'type' => $this->type, - 'jwt' => $this->jwt, - 'payload' => $this->payload, - 'data' => $this->data - ]); - } + $queue = new Client(Event::FUNCTIONS_QUEUE_NAME, $this->connection); - /** - * Schedules the function event and schedules it in the functions worker queue. - * - * @param \DateTime|int $at - * @return void - * @throws \Resque_Exception - * @throws \ResqueScheduler_InvalidTimestampException - */ - public function schedule(DateTime|int $at): void - { - ResqueScheduler::enqueueAt($at, $this->queue, $this->class, [ - 'project' => $this->project, - 'user' => $this->user, - 'function' => $this->function, - 'execution' => $this->execution, + return $queue->enqueue([ 'type' => $this->type, - 'payload' => $this->payload, - 'data' => $this->data + 'execution' => $this->execution, + 'function' => $this->function, + 'data' => $this->data, + 'jwt' => $this->jwt, + 'project' => $this->project, + 'user' => $this->user ]); } } diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index 39e0686da5..88fd0a9a80 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -14,6 +14,7 @@ use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Queue\Client as Worker; use Appwrite\Event\Event; +use Appwrite\Event\Func; use function Swoole\Coroutine\run; @@ -203,6 +204,9 @@ class Schedule extends Action \go(function () use ($delay, $schedules, $scheduleKeys, $pools) { \sleep($delay); // in seconds + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + foreach ($scheduleKeys as $scheduleKey) { // Ensure schedule was not deleted if (!isset($schedules[$scheduleKey])) { @@ -211,20 +215,16 @@ class Schedule extends Action $schedule = $schedules[$scheduleKey]; - $queue = $pools->get('queue')->pop(); + $functions = new Func($connection); - $worker = new Worker(Event::FUNCTIONS_QUEUE_NAME, $queue->getResource()); - $worker - ->enqueue([ - 'type' => 'schedule', - 'value' => [ - 'project' => $schedule['project'], - 'function' => $schedule['function'], - ] - ]); - - $queue->reclaim(); + $functions + ->setType('schedule') + ->setFunction($schedule['function']) + ->setProject($schedule['project']) + ->trigger(); } + + $queue->reclaim(); }); } From 6bf370a0581047b786564d9293e5f6267be7107c Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 19:43:40 +0100 Subject: [PATCH 45/65] QA bug fixes --- app/controllers/api/functions.php | 3 +-- app/worker.php | 2 +- app/workers/builds.php | 4 ++-- app/workers/functions.php | 6 +++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 02c3fce885..7204271d0f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1157,8 +1157,7 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/worker.php b/app/worker.php index dabb91f1d8..564a577c71 100644 --- a/app/worker.php +++ b/app/worker.php @@ -33,7 +33,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) }, ['cache', 'register']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) { - $args = $message->getPayload()['value'] ?? []; + $args = $message->getPayload() ?? []; $project = new Document($args['project'] ?? []); if ($project->isEmpty() || $project->getId() === 'console') { diff --git a/app/workers/builds.php b/app/workers/builds.php index d52329b107..315b13a179 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -106,11 +106,11 @@ class BuildsV1 extends Worker $build->setAttribute('status', 'building'); $build = $dbForProject->updateDocument('builds', $buildId, $build); - $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); - /** Trigger Webhook */ $deploymentModel = new Deployment(); + $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); + $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); $deploymentUpdate ->setProject($project) diff --git a/app/workers/functions.php b/app/workers/functions.php index bad6e7d640..e0c9e64961 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -43,11 +43,11 @@ $execute = function ( ?Document $user = null, string $jwt = null ) use ($executor, $register) { - $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); + /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -209,7 +209,7 @@ $execute = function ( /** Trigger Functions */ $functions - ->setData($data) + ->setData($data ?? '') ->setProject($project) ->setUser($user) ->setEvent('functions.[functionId].executions.[executionId].update') @@ -333,7 +333,7 @@ $server->job() $data = $args['data'] ?? ''; $execution = new Document($args['execution'] ?? []); $user = new Document($args['user'] ?? []); - $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); + // $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt); break; case 'schedule': From 4a92db4dc393fa3e382c308f7f4ff5857d5565aa Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 09:47:46 +0530 Subject: [PATCH 46/65] feat: check async execution --- app/controllers/api/functions.php | 8 +- app/worker.php | 33 ++-- app/workers/functions.php | 99 ++++++------ composer.lock | 2 +- temp/appwrite.json | 21 +++ temp/functions/My Awesome Function/.gitignore | 149 ++++++++++++++++++ temp/functions/My Awesome Function/README.md | 47 ++++++ .../My Awesome Function/package.json | 15 ++ .../My Awesome Function/src/index.js | 46 ++++++ 9 files changed, 352 insertions(+), 68 deletions(-) create mode 100644 temp/appwrite.json create mode 100644 temp/functions/My Awesome Function/.gitignore create mode 100644 temp/functions/My Awesome Function/README.md create mode 100644 temp/functions/My Awesome Function/package.json create mode 100644 temp/functions/My Awesome Function/src/index.js diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7204271d0f..04da1234a1 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1217,10 +1217,10 @@ App::post('/v1/functions/:functionId/executions') // TODO revise this later using route label $usage - ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')); // ms + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')); // ms $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); diff --git a/app/worker.php b/app/worker.php index 564a577c71..110a9d81da 100644 --- a/app/worker.php +++ b/app/worker.php @@ -20,37 +20,37 @@ Server::setResource('register', fn() => $register); Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { $pools = $register->get('pools'); - $dbAdapter = $pools + $database = $pools ->get('console') ->pop() ->getResource() ; - $database = new Database($dbAdapter, $cache); - $database->setNamespace('console'); + $adapter = new Database($database, $cache); + $adapter->setNamespace('console'); - return $database; + return $adapter; }, ['cache', 'register']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) { - $args = $message->getPayload() ?? []; - $project = new Document($args['project'] ?? []); + $payload = $message->getPayload() ?? []; + $project = new Document($payload['project'] ?? []); if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } $pools = $register->get('pools'); - $dbAdapter = $pools + $database = $pools ->get($project->getAttribute('database')) ->pop() ->getResource() ; - $database = new Database($dbAdapter, $cache); - $database->setNamespace('_' . $project->getInternalId()); + $adapter = new Database($database, $cache); + $adapter->setNamespace('_' . $project->getInternalId()); - return $database; + return $adapter; }, ['cache', 'register', 'message', 'dbForConsole']); Server::setResource('cache', function (Registry $register) { @@ -71,19 +71,26 @@ Server::setResource('cache', function (Registry $register) { Server::setResource('functions', function (Registry $register) { $pools = $register->get('pools'); - return new Func($pools->get('queue')->pop()->getResource()); + return new Func( + $pools + ->get('queue') + ->pop() + ->getResource() + ); }, ['register']); Server::setResource('logger', function ($register) { return $register->get('logger'); }, ['register']); +Server::setResource('statsd', function ($register) { + return $register->get('statsd'); +}, ['register']); $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); - $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); $workerNumber = 1; -Runtime::enableCoroutine(SWOOLE_HOOK_ALL); +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); \ No newline at end of file diff --git a/app/workers/functions.php b/app/workers/functions.php index e0c9e64961..02f3932c12 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,13 +2,13 @@ require_once __DIR__ . '/../worker.php'; -use Utopia\Queue; use Utopia\Queue\Message; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Usage\Stats; use Appwrite\Utopia\Response\Model\Execution; +use Domnikl\Statsd\Client; use Executor\Executor; use Utopia\App; use Utopia\CLI\Console; @@ -21,14 +21,16 @@ use Utopia\Database\Query; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; +use Utopia\Queue\Adapter\Swoole; +use Utopia\Queue\Server; Authorization::disable(); Authorization::setDefaultStatus(false); global $connection; global $workerNumber; - -$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); +$adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); +$server = new Server($adapter); $execute = function ( Document $project, @@ -41,12 +43,14 @@ $execute = function ( string $eventData = null, string $data = null, ?Document $user = null, - string $jwt = null -) use ($executor, $register) { + string $jwt = null, + Client $statsd +) { + $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); - + var_dump("Deployment ID : ", $deploymentId); /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -163,6 +167,7 @@ $execute = function ( ]); /** Execute function */ + $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { $executionResponse = $executor->createExecution( projectId: $project->getId(), @@ -223,7 +228,7 @@ $execute = function ( 'executionId' => $execution->getId() ]); $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $execution ); @@ -243,12 +248,11 @@ $execute = function ( ); /** Update usage stats */ - global $register; if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $statsd = $register->get('statsd'); $usage = new Stats($statsd); $usage ->setParam('projectId', $project->getId()) + ->setParam('projectInternalId', $project->getInternalId()) ->setParam('functionId', $function->getId()) ->setParam('executions.{scope}.compute', 1) ->setParam('executionStatus', $execution->getAttribute('status', '')) @@ -259,21 +263,25 @@ $execute = function ( } }; -$adapter = new Queue\Adapter\Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); -$server = new Queue\Server($adapter); - $server->job() ->inject('message') ->inject('dbForProject') ->inject('functions') - ->action(function (Message $message, Database $dbForProject, Func $functions) use ($execute) { - $args = $message->getPayload() ?? []; - $type = $args['type'] ?? ''; - $events = $args['events'] ?? []; - $project = new Document($args['project'] ?? []); - $user = new Document($args['user'] ?? []); - // Where $payload comes from - $payload = json_encode($args['payload'] ?? []); + ->inject('statsd') + ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd) use ($execute) { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + var_dump(json_encode($payload)); + + $type = $payload['type'] ?? ''; + $events = $payload['events'] ?? []; + $project = new Document($payload['project'] ?? []); + $data = $payload['data'] ?? ''; + $user = new Document($payload['user'] ?? []); if ($project->getId() === 'console') { return; @@ -284,38 +292,31 @@ $server->job() */ if (!empty($events)) { $limit = 30; - $sum = $limit; - $total = 0; - $latestDocument = null; + $sum = 30; + $offset = 0; + $functions = []; + /** @var Document[] $functions */ - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - $results = $dbForProject->find('functions', \array_merge($paginationQueries, [ - Query::orderAsc('name') - ])); + while ($sum >= $limit) { + $functions = $dbForProject->find('functions', [ + Query::limit($limit), + Query::offset($offset), + Query::orderAsc('name'), + ]); - $sum = count($results); - $total = $total + $sum; + $sum = \count($functions); + $offset = $offset + $limit; Console::log('Fetched ' . $sum . ' functions...'); - foreach ($results as $function) { + foreach ($functions as $function) { if (!array_intersect($events, $function->getAttribute('events', []))) { continue; } - Console::success('Iterating function: ' . $function->getAttribute('name')); - - // As event, pass first, most verbose event pattern - call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null); - + call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null, $statsd); Console::success('Triggered function: ' . $events[0]); } - - $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } return; @@ -324,20 +325,18 @@ $server->job() /** * Handle Schedule and HTTP execution. */ - $project = new Document($args['project'] ?? []); - $function = new Document($args['function'] ?? []); + $function = new Document($payload['function'] ?? []); + var_dump($function); switch ($type) { case 'http': - $jwt = $args['jwt'] ?? ''; - $data = $args['data'] ?? ''; - $execution = new Document($args['execution'] ?? []); - $user = new Document($args['user'] ?? []); - // $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); - call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt); + $jwt = $payload['jwt'] ?? ''; + $execution = new Document($payload['execution'] ?? []); + $user = new Document($payload['user'] ?? []); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt, $statsd); break; case 'schedule': - call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null, $statsd); break; } }); diff --git a/composer.lock b/composer.lock index 674e42a5ca..d41a4a1cf9 100644 --- a/composer.lock +++ b/composer.lock @@ -5311,5 +5311,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.3.0" } diff --git a/temp/appwrite.json b/temp/appwrite.json new file mode 100644 index 0000000000..643a821415 --- /dev/null +++ b/temp/appwrite.json @@ -0,0 +1,21 @@ +{ + "projectId": "6374484424b42c83155c", + "projectName": "Default", + "functions": [ + { + "$id": "637449ac7066fa0d8cde", + "name": "My Awesome Function", + "runtime": "node-14.5", + "path": "functions/My Awesome Function", + "entrypoint": "src/index.js", + "ignore": [ + "node_modules", + ".npm" + ], + "execute": [], + "events": [], + "schedule": "", + "timeout": 15 + } + ] +} \ No newline at end of file diff --git a/temp/functions/My Awesome Function/.gitignore b/temp/functions/My Awesome Function/.gitignore new file mode 100644 index 0000000000..2551987d6d --- /dev/null +++ b/temp/functions/My Awesome Function/.gitignore @@ -0,0 +1,149 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/node +# Edit at https://www.toptal.com/developers/gitignore?templates=node + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +### Node Patch ### +# Serverless Webpack directories +.webpack/ + +# Optional stylelint cache + +# SvelteKit build / generate output +.svelte-kit + +# End of https://www.toptal.com/developers/gitignore/api/node + +# OS +## Mac +.DS_Store diff --git a/temp/functions/My Awesome Function/README.md b/temp/functions/My Awesome Function/README.md new file mode 100644 index 0000000000..0eb0d2ad04 --- /dev/null +++ b/temp/functions/My Awesome Function/README.md @@ -0,0 +1,47 @@ +# My Awesome Function + +Welcome to the documentation of this function 👋 We strongly recommend keeping this file in sync with your function's logic to make sure anyone can easily understand your function in the future. If you don't need documentation, you can remove this file. + +## 🤖 Documentation + +Simple function similar to typical "hello world" example, but instead, we return a simple JSON that tells everyone how awesome developers are. + + + +_Example input:_ + +This function expects no input + + + +_Example output:_ + + + +```json +{ + "areDevelopersAwesome": true +} +``` + +## 📝 Environment Variables + +List of environment variables used by this cloud function: + +- **APPWRITE_FUNCTION_ENDPOINT** - Endpoint of Appwrite project +- **APPWRITE_FUNCTION_API_KEY** - Appwrite API Key + + +## 🚀 Deployment + +There are two ways of deploying the Appwrite function, both having the same results, but each using a different process. We highly recommend using CLI deployment to achieve the best experience. + +### Using CLI + +Make sure you have [Appwrite CLI](https://appwrite.io/docs/command-line#installation) installed, and you have successfully logged into your Appwrite server. To make sure Appwrite CLI is ready, you can use the command `appwrite client --debug` and it should respond with green text `✓ Success`. + +Make sure you are in the same folder as your `appwrite.json` file and run `appwrite deploy function` to deploy your function. You will be prompted to select which functions you want to deploy. + +### Manual using tar.gz + +Manual deployment has no requirements and uses Appwrite Console to deploy the tag. First, enter the folder of your function. Then, create a tarball of the whole folder and gzip it. After creating `.tar.gz` file, visit Appwrite Console, click on the `Deploy Tag` button and switch to the `Manual` tab. There, set the `entrypoint` to `src/index.js`, and upload the file we just generated. diff --git a/temp/functions/My Awesome Function/package.json b/temp/functions/My Awesome Function/package.json new file mode 100644 index 0000000000..fa252199b9 --- /dev/null +++ b/temp/functions/My Awesome Function/package.json @@ -0,0 +1,15 @@ +{ + "name": "appwrite-function", + "version": "1.0.0", + "description": "", + "main": "src/index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "node-appwrite": "^8.0.0" + } +} diff --git a/temp/functions/My Awesome Function/src/index.js b/temp/functions/My Awesome Function/src/index.js new file mode 100644 index 0000000000..4ed0028cac --- /dev/null +++ b/temp/functions/My Awesome Function/src/index.js @@ -0,0 +1,46 @@ +const sdk = require("node-appwrite"); + +/* + 'req' variable has: + 'headers' - object with request headers + 'payload' - request body data as a string + 'variables' - object with function variables + + 'res' variable has: + 'send(text, status)' - function to return text response. Status code defaults to 200 + 'json(obj, status)' - function to return JSON response. Status code defaults to 200 + + If an error is thrown, a response with code 500 will be returned. +*/ + +module.exports = async function (req, res) { + const client = new sdk.Client(); + + // You can remove services you don't use + const account = new sdk.Account(client); + const avatars = new sdk.Avatars(client); + const database = new sdk.Databases(client); + const functions = new sdk.Functions(client); + const health = new sdk.Health(client); + const locale = new sdk.Locale(client); + const storage = new sdk.Storage(client); + const teams = new sdk.Teams(client); + const users = new sdk.Users(client); + + if ( + !req.variables['APPWRITE_FUNCTION_ENDPOINT'] || + !req.variables['APPWRITE_FUNCTION_API_KEY'] + ) { + console.warn("Environment variables are not set. Function cannot use Appwrite SDK."); + } else { + client + .setEndpoint(req.variables['APPWRITE_FUNCTION_ENDPOINT']) + .setProject(req.variables['APPWRITE_FUNCTION_PROJECT_ID']) + .setKey(req.variables['APPWRITE_FUNCTION_API_KEY']) + .setSelfSigned(true); + } + + res.json({ + areDevelopersAwesome: true, + }); +}; From 3bd3b8d8e9379840a98ab782e32368578d2ba474 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 09:48:11 +0530 Subject: [PATCH 47/65] feat: check async execution --- temp/appwrite.json | 21 --- temp/functions/My Awesome Function/.gitignore | 149 ------------------ temp/functions/My Awesome Function/README.md | 47 ------ .../My Awesome Function/package.json | 15 -- .../My Awesome Function/src/index.js | 46 ------ 5 files changed, 278 deletions(-) delete mode 100644 temp/appwrite.json delete mode 100644 temp/functions/My Awesome Function/.gitignore delete mode 100644 temp/functions/My Awesome Function/README.md delete mode 100644 temp/functions/My Awesome Function/package.json delete mode 100644 temp/functions/My Awesome Function/src/index.js diff --git a/temp/appwrite.json b/temp/appwrite.json deleted file mode 100644 index 643a821415..0000000000 --- a/temp/appwrite.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "projectId": "6374484424b42c83155c", - "projectName": "Default", - "functions": [ - { - "$id": "637449ac7066fa0d8cde", - "name": "My Awesome Function", - "runtime": "node-14.5", - "path": "functions/My Awesome Function", - "entrypoint": "src/index.js", - "ignore": [ - "node_modules", - ".npm" - ], - "execute": [], - "events": [], - "schedule": "", - "timeout": 15 - } - ] -} \ No newline at end of file diff --git a/temp/functions/My Awesome Function/.gitignore b/temp/functions/My Awesome Function/.gitignore deleted file mode 100644 index 2551987d6d..0000000000 --- a/temp/functions/My Awesome Function/.gitignore +++ /dev/null @@ -1,149 +0,0 @@ - -# Created by https://www.toptal.com/developers/gitignore/api/node -# Edit at https://www.toptal.com/developers/gitignore?templates=node - -### Node ### -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local -.env.local - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp and cache directory -.temp - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# yarn v2 -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.* - -### Node Patch ### -# Serverless Webpack directories -.webpack/ - -# Optional stylelint cache - -# SvelteKit build / generate output -.svelte-kit - -# End of https://www.toptal.com/developers/gitignore/api/node - -# OS -## Mac -.DS_Store diff --git a/temp/functions/My Awesome Function/README.md b/temp/functions/My Awesome Function/README.md deleted file mode 100644 index 0eb0d2ad04..0000000000 --- a/temp/functions/My Awesome Function/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# My Awesome Function - -Welcome to the documentation of this function 👋 We strongly recommend keeping this file in sync with your function's logic to make sure anyone can easily understand your function in the future. If you don't need documentation, you can remove this file. - -## 🤖 Documentation - -Simple function similar to typical "hello world" example, but instead, we return a simple JSON that tells everyone how awesome developers are. - - - -_Example input:_ - -This function expects no input - - - -_Example output:_ - - - -```json -{ - "areDevelopersAwesome": true -} -``` - -## 📝 Environment Variables - -List of environment variables used by this cloud function: - -- **APPWRITE_FUNCTION_ENDPOINT** - Endpoint of Appwrite project -- **APPWRITE_FUNCTION_API_KEY** - Appwrite API Key - - -## 🚀 Deployment - -There are two ways of deploying the Appwrite function, both having the same results, but each using a different process. We highly recommend using CLI deployment to achieve the best experience. - -### Using CLI - -Make sure you have [Appwrite CLI](https://appwrite.io/docs/command-line#installation) installed, and you have successfully logged into your Appwrite server. To make sure Appwrite CLI is ready, you can use the command `appwrite client --debug` and it should respond with green text `✓ Success`. - -Make sure you are in the same folder as your `appwrite.json` file and run `appwrite deploy function` to deploy your function. You will be prompted to select which functions you want to deploy. - -### Manual using tar.gz - -Manual deployment has no requirements and uses Appwrite Console to deploy the tag. First, enter the folder of your function. Then, create a tarball of the whole folder and gzip it. After creating `.tar.gz` file, visit Appwrite Console, click on the `Deploy Tag` button and switch to the `Manual` tab. There, set the `entrypoint` to `src/index.js`, and upload the file we just generated. diff --git a/temp/functions/My Awesome Function/package.json b/temp/functions/My Awesome Function/package.json deleted file mode 100644 index fa252199b9..0000000000 --- a/temp/functions/My Awesome Function/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "appwrite-function", - "version": "1.0.0", - "description": "", - "main": "src/index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "node-appwrite": "^8.0.0" - } -} diff --git a/temp/functions/My Awesome Function/src/index.js b/temp/functions/My Awesome Function/src/index.js deleted file mode 100644 index 4ed0028cac..0000000000 --- a/temp/functions/My Awesome Function/src/index.js +++ /dev/null @@ -1,46 +0,0 @@ -const sdk = require("node-appwrite"); - -/* - 'req' variable has: - 'headers' - object with request headers - 'payload' - request body data as a string - 'variables' - object with function variables - - 'res' variable has: - 'send(text, status)' - function to return text response. Status code defaults to 200 - 'json(obj, status)' - function to return JSON response. Status code defaults to 200 - - If an error is thrown, a response with code 500 will be returned. -*/ - -module.exports = async function (req, res) { - const client = new sdk.Client(); - - // You can remove services you don't use - const account = new sdk.Account(client); - const avatars = new sdk.Avatars(client); - const database = new sdk.Databases(client); - const functions = new sdk.Functions(client); - const health = new sdk.Health(client); - const locale = new sdk.Locale(client); - const storage = new sdk.Storage(client); - const teams = new sdk.Teams(client); - const users = new sdk.Users(client); - - if ( - !req.variables['APPWRITE_FUNCTION_ENDPOINT'] || - !req.variables['APPWRITE_FUNCTION_API_KEY'] - ) { - console.warn("Environment variables are not set. Function cannot use Appwrite SDK."); - } else { - client - .setEndpoint(req.variables['APPWRITE_FUNCTION_ENDPOINT']) - .setProject(req.variables['APPWRITE_FUNCTION_PROJECT_ID']) - .setKey(req.variables['APPWRITE_FUNCTION_API_KEY']) - .setSelfSigned(true); - } - - res.json({ - areDevelopersAwesome: true, - }); -}; From 586fd46bcdb84df6d4450eb3d6eacaaa34427e39 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 09:49:35 +0530 Subject: [PATCH 48/65] feat: check async execution --- app/workers/functions.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 02f3932c12..67fb716e54 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -279,9 +279,11 @@ $server->job() $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; - $project = new Document($payload['project'] ?? []); $data = $payload['data'] ?? ''; + $project = new Document($payload['project'] ?? []); + $function = new Document($payload['function'] ?? []); $user = new Document($payload['user'] ?? []); + var_dump("Function : ", $function); if ($project->getId() === 'console') { return; @@ -325,9 +327,6 @@ $server->job() /** * Handle Schedule and HTTP execution. */ - $function = new Document($payload['function'] ?? []); - var_dump($function); - switch ($type) { case 'http': $jwt = $payload['jwt'] ?? ''; From 5abe9ad73cfb46a5120ea779aa73e312643202de Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 10:10:34 +0530 Subject: [PATCH 49/65] feat: refactor execute function to a resource --- app/workers/functions.php | 503 ++++++++++++++++++++------------------ 1 file changed, 271 insertions(+), 232 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 67fb716e54..a34f0aec99 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -32,243 +32,246 @@ global $workerNumber; $adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); $server = new Server($adapter); -$execute = function ( - Document $project, - Document $function, - Database $dbForProject, - Func $functions, - string $trigger, - string $executionId = null, - string $event = null, - string $eventData = null, - string $data = null, - ?Document $user = null, - string $jwt = null, - Client $statsd -) { +Server::setResource('execute', function () { + return function ( + Document $project, + Document $function, + Database $dbForProject, + Func $functions, + string $trigger, + string $executionId = null, + string $event = null, + string $eventData = null, + string $data = null, + ?Document $user = null, + string $jwt = null, + Client $statsd + ) { - $user ??= new Document(); - $functionId = $function->getId(); - $deploymentId = $function->getAttribute('deployment', ''); - var_dump("Deployment ID : ", $deploymentId); - - /** Check if deployment exists */ - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - if ($deployment->getAttribute('resourceId') !== $functionId) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function'); - } - - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function'); - } - - /** Check if build has exists */ - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - if ($build->isEmpty()) { - throw new Exception('Build not found'); - } - - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready'); - } - - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); - - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); + $user ??= new Document(); + $functionId = $function->getId(); + $deploymentId = $function->getAttribute('deployment', ''); + var_dump("Deployment ID : ", $deploymentId); + + /** Check if deployment exists */ + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if ($deployment->getAttribute('resourceId') !== $functionId) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready'); - } - - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); - - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); + + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { - $carry[$var->getAttribute('key')] = $var->getAttribute('value'); - return $carry; - }, []); - - /** Collect environment variables */ - $vars = \array_merge($vars, [ - 'APPWRITE_FUNCTION_ID' => $functionId, - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, - 'APPWRITE_FUNCTION_TRIGGER' => $trigger, - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_EVENT' => $event ?? '', - 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', - 'APPWRITE_FUNCTION_DATA' => $data ?? '', - 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', - 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', - ]); - - /** Execute function */ - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - try { - $executionResponse = $executor->createExecution( - projectId: $project->getId(), - deploymentId: $deployment->getId(), - payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', - variables: $vars, - timeout: $function->getAttribute('timeout', 0), - image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), - entrypoint: $deployment->getAttribute('entrypoint', ''), - ); - - /** Update execution status */ - $execution - ->setAttribute('status', $executionResponse['status']) - ->setAttribute('statusCode', $executionResponse['statusCode']) - ->setAttribute('response', $executionResponse['response']) - ->setAttribute('stdout', $executionResponse['stdout']) - ->setAttribute('stderr', $executionResponse['stderr']) - ->setAttribute('duration', $executionResponse['duration']); - } catch (\Throwable $th) { - $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); - $execution - ->setAttribute('duration', (float)$interval->format('%s.%f')) - ->setAttribute('status', 'failed') - ->setAttribute('statusCode', $th->getCode()) - ->setAttribute('stderr', $th->getMessage()); - Console::error($th->getMessage()); - } - - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - /** Trigger Webhook */ - $executionModel = new Execution(); - $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); - $executionUpdate - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) - ->trigger(); - - /** Trigger Functions */ - $functions - ->setData($data ?? '') - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->trigger(); - - /** Trigger realtime event */ - $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ - 'functionId' => $function->getId(), - 'executionId' => $execution->getId() - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $execution - ); - Realtime::send( - projectId: 'console', - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - /** Update usage stats */ - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $usage = new Stats($statsd); - $usage - ->setParam('projectId', $project->getId()) - ->setParam('projectInternalId', $project->getInternalId()) + + /** Check if build has exists */ + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + if ($build->isEmpty()) { + throw new Exception('Build not found'); + } + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready'); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready'); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $carry[$var->getAttribute('key')] = $var->getAttribute('value'); + return $carry; + }, []); + + /** Collect environment variables */ + $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_ID' => $functionId, + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, + 'APPWRITE_FUNCTION_TRIGGER' => $trigger, + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_FUNCTION_EVENT' => $event ?? '', + 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', + 'APPWRITE_FUNCTION_DATA' => $data ?? '', + 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', + 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', + ]); + + /** Execute function */ + $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + try { + $executionResponse = $executor->createExecution( + projectId: $project->getId(), + deploymentId: $deployment->getId(), + payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', + variables: $vars, + timeout: $function->getAttribute('timeout', 0), + image: $runtime['image'], + source: $build->getAttribute('outputPath', ''), + entrypoint: $deployment->getAttribute('entrypoint', ''), + ); + + /** Update execution status */ + $execution + ->setAttribute('status', $executionResponse['status']) + ->setAttribute('statusCode', $executionResponse['statusCode']) + ->setAttribute('response', $executionResponse['response']) + ->setAttribute('stdout', $executionResponse['stdout']) + ->setAttribute('stderr', $executionResponse['stderr']) + ->setAttribute('duration', $executionResponse['duration']); + } catch (\Throwable $th) { + $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); + $execution + ->setAttribute('duration', (float)$interval->format('%s.%f')) + ->setAttribute('status', 'failed') + ->setAttribute('statusCode', $th->getCode()) + ->setAttribute('stderr', $th->getMessage()); + Console::error($th->getMessage()); + } + + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + /** Trigger Webhook */ + $executionModel = new Execution(); + $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); + $executionUpdate + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')) - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ->submit(); - } -}; + ->setParam('executionId', $execution->getId()) + ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) + ->trigger(); + + /** Trigger Functions */ + $functions + ->setData($data ?? '') + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) + ->trigger(); + + /** Trigger realtime event */ + $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ + 'functionId' => $function->getId(), + 'executionId' => $execution->getId() + ]); + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $execution + ); + Realtime::send( + projectId: 'console', + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + Realtime::send( + projectId: $project->getId(), + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + /** Update usage stats */ + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $usage = new Stats($statsd); + $usage + ->setParam('projectId', $project->getId()) + ->setParam('projectInternalId', $project->getInternalId()) + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')) + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit(); + } + }; +}); $server->job() ->inject('message') ->inject('dbForProject') ->inject('functions') ->inject('statsd') - ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd) use ($execute) { + ->inject('execute') + ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd, callable $execute) { $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -276,7 +279,6 @@ $server->job() } var_dump(json_encode($payload)); - $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; $data = $payload['data'] ?? ''; @@ -298,7 +300,6 @@ $server->job() $offset = 0; $functions = []; /** @var Document[] $functions */ - while ($sum >= $limit) { $functions = $dbForProject->find('functions', [ Query::limit($limit), @@ -316,7 +317,19 @@ $server->job() continue; } Console::success('Iterating function: ' . $function->getAttribute('name')); - call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null, $statsd); + $execute( + statsd: $statsd, + dbForProject: $dbForProject, + project: $project, + function: $function, + trigger: 'event', + event: $events[0], + eventData: $payload, + user: $user, + data: null, + executionId: null, + jwt: null + ); Console::success('Triggered function: ' . $events[0]); } } @@ -332,10 +345,36 @@ $server->job() $jwt = $payload['jwt'] ?? ''; $execution = new Document($payload['execution'] ?? []); $user = new Document($payload['user'] ?? []); - call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt, $statsd); + $execute( + project: $project, + function: $function, + dbForProject: $dbForProject, + functions: $functions, + trigger: 'http', + executionId: $execution->getId(), + event: null, + eventData: null, + data: $data, + user: $user, + jwt: $jwt, + statsd: $statsd, + ); break; case 'schedule': - call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null, $statsd); + $execute( + project: $project, + function: $function, + dbForProject: $dbForProject, + functions: $functions, + trigger: 'http', + executionId: null, + event: null, + eventData: null, + data: null, + user: null, + jwt: null, + statsd: $statsd, + ); break; } }); From 8b0a78bb36672ec986df87031e4face4a34ce3ba Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 11:00:57 +0530 Subject: [PATCH 50/65] fix: function events and linter --- app/controllers/api/functions.php | 3 +- app/controllers/shared/api.php | 15 ++-- app/worker.php | 4 +- app/workers/functions.php | 79 ++++++++++--------- src/Appwrite/Event/Func.php | 36 +++++++-- .../Utopia/Response/Model/Execution.php | 1 - 6 files changed, 78 insertions(+), 60 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 04da1234a1..b695d19429 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1157,7 +1157,8 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user); + ->setUser($user) + ->trigger(); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 58dd2f16ef..a12016606f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -129,9 +129,9 @@ App::init() } } - /* - * Background Jobs - */ + /* + * Background Jobs + */ $events ->setEvent($route->getLabel('event', '')) ->setProject($project) @@ -265,12 +265,9 @@ App::shutdown() * Trigger functions. */ $functions - ->setData(\json_encode($events->getPayload())) - ->setProject($events->getProject()) - ->setUser($events->getUser()) - ->setEvent($events->getEvent()) - ->setParam('functionId', $events->getParam('functionId')) - ->setParam('executionId', $events->getParam('executionId')) + ->from($events) + ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + ->setClass(Event::FUNCTIONS_CLASS_NAME) ->trigger(); /** diff --git a/app/worker.php b/app/worker.php index 110a9d81da..60e93b2518 100644 --- a/app/worker.php +++ b/app/worker.php @@ -76,7 +76,7 @@ Server::setResource('functions', function (Registry $register) { ->get('queue') ->pop() ->getResource() - ); + ); }, ['register']); Server::setResource('logger', function ($register) { @@ -93,4 +93,4 @@ $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); $workerNumber = 1; -Runtime::enableCoroutine(SWOOLE_HOOK_ALL); \ No newline at end of file +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); diff --git a/app/workers/functions.php b/app/workers/functions.php index a34f0aec99..9d1dc0372d 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -52,37 +52,37 @@ Server::setResource('execute', function () { $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); var_dump("Deployment ID : ", $deploymentId); - + /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); - + if ($deployment->getAttribute('resourceId') !== $functionId) { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - + if ($deployment->isEmpty()) { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - + /** Check if build has exists */ $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($build->isEmpty()) { throw new Exception('Build not found'); } - + if ($build->getAttribute('status') !== 'ready') { throw new Exception('Build not ready'); } - + /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); - + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } - + $runtime = $runtimes[$function->getAttribute('runtime')]; - + /** Create execution or update execution status */ $execution = $dbForProject->getDocument('executions', $executionId ?? ''); if ($execution->isEmpty()) { @@ -100,27 +100,27 @@ Server::setResource('execute', function () { 'duration' => 0.0, 'search' => implode(' ', [$functionId, $executionId]), ])); - + if ($execution->isEmpty()) { throw new Exception('Failed to create or read execution'); } } $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - + if ($build->getAttribute('status') !== 'ready') { throw new Exception('Build not ready'); } - + /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); - + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } - + $runtime = $runtimes[$function->getAttribute('runtime')]; - + /** Create execution or update execution status */ $execution = $dbForProject->getDocument('executions', $executionId ?? ''); if ($execution->isEmpty()) { @@ -138,19 +138,19 @@ Server::setResource('execute', function () { 'duration' => 0.0, 'search' => implode(' ', [$functionId, $executionId]), ])); - + if ($execution->isEmpty()) { throw new Exception('Failed to create or read execution'); } } $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - + $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value'); return $carry; }, []); - + /** Collect environment variables */ $vars = \array_merge($vars, [ 'APPWRITE_FUNCTION_ID' => $functionId, @@ -166,7 +166,7 @@ Server::setResource('execute', function () { 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', ]); - + /** Execute function */ $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { @@ -180,7 +180,7 @@ Server::setResource('execute', function () { source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); - + /** Update execution status */ $execution ->setAttribute('status', $executionResponse['status']) @@ -198,9 +198,9 @@ Server::setResource('execute', function () { ->setAttribute('stderr', $th->getMessage()); Console::error($th->getMessage()); } - + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - + /** Trigger Webhook */ $executionModel = new Execution(); $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); @@ -212,7 +212,7 @@ Server::setResource('execute', function () { ->setParam('executionId', $execution->getId()) ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) ->trigger(); - + /** Trigger Functions */ $functions ->setData($data ?? '') @@ -222,7 +222,7 @@ Server::setResource('execute', function () { ->setParam('functionId', $function->getId()) ->setParam('executionId', $execution->getId()) ->trigger(); - + /** Trigger realtime event */ $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ 'functionId' => $function->getId(), @@ -247,7 +247,7 @@ Server::setResource('execute', function () { channels: $target['channels'], roles: $target['roles'] ); - + /** Update usage stats */ if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $usage = new Stats($statsd); @@ -282,6 +282,7 @@ $server->job() $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; $data = $payload['data'] ?? ''; + $eventData = $payload['payload'] ?? ''; $project = new Document($payload['project'] ?? []); $function = new Document($payload['function'] ?? []); $user = new Document($payload['user'] ?? []); @@ -320,14 +321,14 @@ $server->job() $execute( statsd: $statsd, dbForProject: $dbForProject, - project: $project, + project: $project, function: $function, - trigger: 'event', - event: $events[0], - eventData: $payload, + trigger: 'event', + event: $events[0], + eventData: $eventData, user: $user, - data: null, - executionId: null, + data: null, + executionId: null, jwt: null ); Console::success('Triggered function: ' . $events[0]); @@ -346,15 +347,15 @@ $server->job() $execution = new Document($payload['execution'] ?? []); $user = new Document($payload['user'] ?? []); $execute( - project: $project, + project: $project, function: $function, dbForProject: $dbForProject, functions: $functions, trigger: 'http', - executionId: $execution->getId(), - event: null, + executionId: $execution->getId(), + event: null, eventData: null, - data: $data, + data: $data, user: $user, jwt: $jwt, statsd: $statsd, @@ -362,15 +363,15 @@ $server->job() break; case 'schedule': $execute( - project: $project, + project: $project, function: $function, dbForProject: $dbForProject, functions: $functions, trigger: 'http', - executionId: null, - event: null, + executionId: null, + event: null, eventData: null, - data: null, + data: null, user: null, jwt: null, statsd: $statsd, diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 514eb5df74..9fdcc4f8f3 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -145,16 +145,36 @@ class Func extends Event */ public function trigger(): string|bool { - $queue = new Client(Event::FUNCTIONS_QUEUE_NAME, $this->connection); + $client = new Client($this->queue, $this->connection); - return $queue->enqueue([ - 'type' => $this->type, - 'execution' => $this->execution, - 'function' => $this->function, - 'data' => $this->data, - 'jwt' => $this->jwt, + return $client->enqueue([ 'project' => $this->project, - 'user' => $this->user + 'user' => $this->user, + 'function' => $this->function, + 'execution' => $this->execution, + 'type' => $this->type, + 'jwt' => $this->jwt, + 'payload' => '', + 'events' => Event::generateEvents($this->getEvent(), $this->getParams()), + 'data' => $this->data, ]); } + + /** + * Generate a function event from a base event + * + * @param Event $event + * + * @return self + * + */ + public function from(Event $event): self + { + $this->project = $event->getProject(); + $this->user = $event->getUser(); + $this->payload = $event->getPayload(); + $this->event = $event->getEvent(); + $this->params = $event->getParams(); + return $this; + } } diff --git a/src/Appwrite/Utopia/Response/Model/Execution.php b/src/Appwrite/Utopia/Response/Model/Execution.php index 987a140dfb..13011a24b7 100644 --- a/src/Appwrite/Utopia/Response/Model/Execution.php +++ b/src/Appwrite/Utopia/Response/Model/Execution.php @@ -6,7 +6,6 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use Utopia\Database\Role; - class Execution extends Model { public function __construct() From e976bd3cb1ea022f0323ab2a46211dbaa5adcc64 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 11:22:19 +0530 Subject: [PATCH 51/65] fix: event triggers --- app/workers/functions.php | 2 -- src/Appwrite/Event/Event.php | 2 -- src/Appwrite/Event/Func.php | 4 +++- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 9d1dc0372d..e2a31afd75 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -51,7 +51,6 @@ Server::setResource('execute', function () { $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); - var_dump("Deployment ID : ", $deploymentId); /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -286,7 +285,6 @@ $server->job() $project = new Document($payload['project'] ?? []); $function = new Document($payload['function'] ?? []); $user = new Document($payload['user'] ?? []); - var_dump("Function : ", $function); if ($project->getId() === 'console') { return; diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 222cf59444..6e3401e11b 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -337,8 +337,6 @@ class Event default => false }; - - return [ 'type' => $type, 'resource' => $resource, diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 9fdcc4f8f3..b114ca4b59 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -147,6 +147,8 @@ class Func extends Event { $client = new Client($this->queue, $this->connection); + $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; + return $client->enqueue([ 'project' => $this->project, 'user' => $this->user, @@ -155,7 +157,7 @@ class Func extends Event 'type' => $this->type, 'jwt' => $this->jwt, 'payload' => '', - 'events' => Event::generateEvents($this->getEvent(), $this->getParams()), + 'events' => $events, 'data' => $this->data, ]); } From dbb05c46ba734bd438b291257cdeefdbf071eeb6 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 11:38:01 +0530 Subject: [PATCH 52/65] fix: event triggers --- app/worker.php | 2 +- app/workers/functions.php | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/worker.php b/app/worker.php index 60e93b2518..c9074f6e20 100644 --- a/app/worker.php +++ b/app/worker.php @@ -69,7 +69,7 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); -Server::setResource('functions', function (Registry $register) { +Server::setResource('queueForFunctions', function (Registry $register) { $pools = $register->get('pools'); return new Func( $pools diff --git a/app/workers/functions.php b/app/workers/functions.php index e2a31afd75..bfa4421449 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -37,7 +37,7 @@ Server::setResource('execute', function () { Document $project, Document $function, Database $dbForProject, - Func $functions, + Func $queueForFunctions, string $trigger, string $executionId = null, string $event = null, @@ -213,7 +213,7 @@ Server::setResource('execute', function () { ->trigger(); /** Trigger Functions */ - $functions + $queueForFunctions ->setData($data ?? '') ->setProject($project) ->setUser($user) @@ -267,10 +267,10 @@ Server::setResource('execute', function () { $server->job() ->inject('message') ->inject('dbForProject') - ->inject('functions') + ->inject('queueForFunctions') ->inject('statsd') ->inject('execute') - ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd, callable $execute) { + ->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute) { $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -321,6 +321,7 @@ $server->job() dbForProject: $dbForProject, project: $project, function: $function, + queueForFunctions: $queueForFunctions, trigger: 'event', event: $events[0], eventData: $eventData, @@ -348,7 +349,7 @@ $server->job() project: $project, function: $function, dbForProject: $dbForProject, - functions: $functions, + queueForFunctions: $queueForFunctions, trigger: 'http', executionId: $execution->getId(), event: null, @@ -364,7 +365,7 @@ $server->job() project: $project, function: $function, dbForProject: $dbForProject, - functions: $functions, + queueForFunctions: $queueForFunctions, trigger: 'http', executionId: null, event: null, From 4e3bcd6cd0dbb43308187bf973519ab9d335f40a Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 10:41:57 +0200 Subject: [PATCH 53/65] Authorization::skip added --- app/controllers/api/functions.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b695d19429..7c9bd4feaf 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -484,7 +484,7 @@ App::put('/v1/functions/:functionId') ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $eventsInstance->setParam('functionId', $function->getId()); @@ -1157,8 +1157,7 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1218,10 +1217,10 @@ App::post('/v1/functions/:functionId/executions') // TODO revise this later using route label $usage - ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')); // ms + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')); // ms $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); From 6b88f66d14939b9ad061709611124287512e3aae Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 10:46:47 +0200 Subject: [PATCH 54/65] removed pools injection in functions controller --- app/controllers/api/functions.php | 4 +--- app/init.php | 17 ----------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7c9bd4feaf..d64f302636 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -14,7 +14,6 @@ use Utopia\Database\Permission; use Utopia\Database\Role; use Utopia\Database\Validator\UID; use Appwrite\Usage\Stats; -use Utopia\Pools\Group; use Utopia\Storage\Device; use Utopia\Storage\Validator\File; use Utopia\Storage\Validator\FileExt; @@ -1060,9 +1059,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('events') ->inject('usage') ->inject('mode') - ->inject('pools') ->inject('functions') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools, Func $functions) { + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $functions) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); diff --git a/app/init.php b/app/init.php index 19cea371df..a6c4b89c45 100644 --- a/app/init.php +++ b/app/init.php @@ -1026,23 +1026,6 @@ App::setResource('console', function () { ]); }, []); -App::setResource('queue', function () { - $fallbackForRedis = AppwriteURL::unparse([ - 'scheme' => 'redis', - 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), - 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), - 'user' => App::getEnv('_APP_REDIS_USER', ''), - 'pass' => App::getEnv('_APP_REDIS_PASS', ''), - ]); - - $connection = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); - - $dsns = explode(',', $connection ?? ''); - $dsn = explode('=', $dsns[0]); - $dsn = $dsn[1] ?? ''; - return new DSN($dsn); -}, []); - App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; From ac70537ece6730d7359e4cdc1ad60971fda29942 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 10:50:24 +0200 Subject: [PATCH 55/65] resetting workers-number --- app/worker.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/worker.php b/app/worker.php index c9074f6e20..5949bd936f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -91,6 +91,5 @@ $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -$workerNumber = 1; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); From 0d61a7fcf5a6511b82b705703ea496c937cc06b2 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 11:00:11 +0200 Subject: [PATCH 56/65] resetting workers-number --- app/worker.php | 1 - docker-compose.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/app/worker.php b/app/worker.php index 5949bd936f..3cdfb81aab 100644 --- a/app/worker.php +++ b/app/worker.php @@ -89,7 +89,6 @@ Server::setResource('statsd', function ($register) { $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); - $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); Runtime::enableCoroutine(SWOOLE_HOOK_ALL); diff --git a/docker-compose.yml b/docker-compose.yml index 4bef6b1a01..22ed89bed4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -500,6 +500,7 @@ services: - _APP_USAGE_STATS - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD + - _APP_WORKER_PER_CORE appwrite-worker-mails: entrypoint: worker-mails From 44875e2d18b44804ceee8814aaaac1be4184835f Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 11:33:11 +0100 Subject: [PATCH 57/65] QA review changes --- app/cli.php | 2 +- app/controllers/api/functions.php | 18 +++++++++--------- app/controllers/shared/api.php | 8 +++----- app/init.php | 2 +- app/workers/builds.php | 22 +++++++++++++--------- app/workers/functions.php | 3 ++- src/Appwrite/Platform/Tasks/Schedule.php | 4 +--- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/app/cli.php b/app/cli.php index 544148c8d5..13709b9b57 100644 --- a/app/cli.php +++ b/app/cli.php @@ -110,7 +110,7 @@ CLI::setResource('influxdb', function (Registry $register) { return $database; }, ['register']); -CLI::setResource('functions', function (Group $pools) { +CLI::setResource('queueForFunctions', function (Group $pools) { return new Func($pools->get('queue')->pop()->getResource()); }, ['pools']); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d64f302636..86981b0071 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -469,13 +469,13 @@ App::put('/v1/functions/:functionId') 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); /** * In case we want to clear the schedule */ if (!empty($function->getAttribute('deployment'))) { - $schedule->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); + $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); } $schedule @@ -537,7 +537,7 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') 'deployment' => $deployment->getId() ]))); - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $active = !empty($function->getAttribute('schedule')); @@ -588,7 +588,7 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) @@ -790,7 +790,7 @@ App::post('/v1/functions/:functionId/deployments') * TODO Should we update also the function collection with the scheduleUpdatedAt attr? */ - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $active = !empty($function->getAttribute('schedule')); @@ -1059,8 +1059,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('events') ->inject('usage') ->inject('mode') - ->inject('functions') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $functions) { + ->inject('queueForFunctions') + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $queueForFunctions) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1148,7 +1148,7 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $functions + $queueForFunctions ->setType('http') ->setExecution($execution) ->setFunction($function) @@ -1162,7 +1162,7 @@ App::post('/v1/functions/:functionId/executions') ->dynamic($execution, Response::MODEL_EXECUTION); } - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value') ?? ''; return $carry; }, []); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a12016606f..5624486755 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -252,8 +252,8 @@ App::shutdown() ->inject('database') ->inject('mode') ->inject('dbForProject') - ->inject('functions') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $functions) use ($parseLabel) { + ->inject('queueForFunctions') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $queueForFunctions) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -264,10 +264,8 @@ App::shutdown() /** * Trigger functions. */ - $functions + $queueForFunctions ->from($events) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) - ->setClass(Event::FUNCTIONS_CLASS_NAME) ->trigger(); /** diff --git a/app/init.php b/app/init.php index a6c4b89c45..af3b05965d 100644 --- a/app/init.php +++ b/app/init.php @@ -844,7 +844,7 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); -App::setResource('functions', function (Group $pools) { +App::setResource('queueForFunctions', function (Group $pools) { return new Func($pools->get('queue')->pop()->getResource()); }, ['pools']); App::setResource('usage', function ($register) { diff --git a/app/workers/builds.php b/app/workers/builds.php index 315b13a179..478c5c5423 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -15,6 +15,7 @@ use Utopia\Database\ID; use Utopia\Storage\Storage; use Utopia\Database\Document; use Utopia\Config\Config; +use Utopia\Database\Validator\Authorization; require_once __DIR__ . '/../init.php'; @@ -156,7 +157,7 @@ class BuildsV1 extends Worker $source = $deployment->getAttribute('path'); - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value'); return $carry; }, []); @@ -192,6 +193,8 @@ class BuildsV1 extends Worker Console::success("Build id: $buildId created"); + $function->setAttribute('scheduleUpdatedAt', DateTime::now()); + /** Set auto deploy */ if ($deployment->getAttribute('activate') === true) { $function->setAttribute('deployment', $deployment->getId()); @@ -199,14 +202,15 @@ class BuildsV1 extends Worker } /** Update function schedule */ - // TODO: @Meldiron refactor scheduler here - /* - $schedule = $function->getAttribute('schedule', ''); - $cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null; - $next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? DateTime::format($cron->getNextRunDate()) : null; - $function->setAttribute('scheduleNext', $next); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - */ + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); + + $schedule + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + + + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); diff --git a/app/workers/functions.php b/app/workers/functions.php index bfa4421449..68b9c21bf6 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -29,6 +29,7 @@ Authorization::setDefaultStatus(false); global $connection; global $workerNumber; + $adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); $server = new Server($adapter); @@ -145,7 +146,7 @@ Server::setResource('execute', function () { $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value'); return $carry; }, []); diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index 88fd0a9a80..dc86f9293e 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -3,17 +3,15 @@ namespace Appwrite\Platform\Tasks; use Cron\CronExpression; +use Swoole\Timer; use Utopia\App; use Utopia\Platform\Action; use Utopia\CLI\Console; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Query; -use Swoole\Timer; use Utopia\Database\Database; use Utopia\Pools\Group; -use Utopia\Queue\Client as Worker; -use Appwrite\Event\Event; use Appwrite\Event\Func; use function Swoole\Coroutine\run; From 9e4a65605c55b00015da94103cb8985028190efc Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 11:40:41 +0100 Subject: [PATCH 58/65] Simplify worker --- app/worker.php | 56 ++++++++++++++++++++++++++++++++++++++- app/workers/functions.php | 52 ------------------------------------ bin/worker-functions | 2 +- composer.json | 2 +- 4 files changed, 57 insertions(+), 55 deletions(-) diff --git a/app/worker.php b/app/worker.php index 3cdfb81aab..7bc1139251 100644 --- a/app/worker.php +++ b/app/worker.php @@ -7,12 +7,17 @@ use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; +use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Queue\Adapter\Swoole; use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; +use Utopia\Logger\Log; + +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); global $register; @@ -91,4 +96,53 @@ $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -Runtime::enableCoroutine(SWOOLE_HOOK_ALL); +if(empty(App::getEnv('QUEUE'))) { + throw new Exception('Please configure "QUEUE" environemnt variable.'); +} + +$adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE')); +$server = new Server($adapter); + +$server + ->error() + ->inject('error') + ->inject('logger') + ->inject('register') + ->action(function ($error, $logger, $register) { + + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); + + if ($error instanceof PDOException) { + throw $error; + } + + if ($error->getCode() >= 500 || $error->getCode() === 0) { + $log = new Log(); + + $log->setNamespace("appwrite-worker"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + $log->setAction('appwrite-worker-functions'); + $log->addTag('verboseType', get_class($error)); + $log->addTag('code', $error->getCode()); + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + $log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $logger->addLog($log); + } + + Console::error('[Error] Type: ' . get_class($error)); + Console::error('[Error] Message: ' . $error->getMessage()); + Console::error('[Error] File: ' . $error->getFile()); + Console::error('[Error] Line: ' . $error->getLine()); + + $register->get('pools')->reclaim(); + }); \ No newline at end of file diff --git a/app/workers/functions.php b/app/workers/functions.php index 68b9c21bf6..f1e472b471 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -20,19 +20,11 @@ use Utopia\Database\Permission; use Utopia\Database\Query; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; -use Utopia\Logger\Log; -use Utopia\Queue\Adapter\Swoole; use Utopia\Queue\Server; Authorization::disable(); Authorization::setDefaultStatus(false); -global $connection; -global $workerNumber; - -$adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); -$server = new Server($adapter); - Server::setResource('execute', function () { return function ( Document $project, @@ -380,49 +372,5 @@ $server->job() } }); -$server - ->error() - ->inject('error') - ->inject('logger') - ->inject('register') - ->action(function ($error, $logger, $register) { - - $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - - if ($error instanceof PDOException) { - throw $error; - } - - if ($error->getCode() >= 500 || $error->getCode() === 0) { - $log = new Log(); - - $log->setNamespace("appwrite-worker"); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - $log->setAction('appwrite-worker-functions'); - $log->addTag('verboseType', get_class($error)); - $log->addTag('code', $error->getCode()); - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - $log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - $logger->addLog($log); - } - - Console::error('[Error] Type: ' . get_class($error)); - Console::error('[Error] Message: ' . $error->getMessage()); - Console::error('[Error] File: ' . $error->getFile()); - Console::error('[Error] Line: ' . $error->getLine()); - - $register->get('pools')->reclaim(); - }); - $server->workerStart(); $server->start(); diff --git a/bin/worker-functions b/bin/worker-functions index b22ee65d39..687f9fd0cd 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file +QUEUE=v1-functions php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 08ede65dcf..ffd9c6b6b9 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/platform": "0.3.*", "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", - "utopia-php/registry": "dev-feat-allow-params as 0.5.0", + "utopia-php/registry": "0.5.*", "utopia-php/storage": "0.11.*", "utopia-php/swoole": "0.5.*", "utopia-php/websocket": "0.1.0", From 390f407eb5cda77e6595c3bb4183a5babf0bb9ac Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 11:54:21 +0100 Subject: [PATCH 59/65] Linter fixes --- app/worker.php | 4 ++-- app/workers/builds.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/worker.php b/app/worker.php index 7bc1139251..eb7e570007 100644 --- a/app/worker.php +++ b/app/worker.php @@ -96,7 +96,7 @@ $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -if(empty(App::getEnv('QUEUE'))) { +if (empty(App::getEnv('QUEUE'))) { throw new Exception('Please configure "QUEUE" environemnt variable.'); } @@ -145,4 +145,4 @@ $server Console::error('[Error] Line: ' . $error->getLine()); $register->get('pools')->reclaim(); - }); \ No newline at end of file + }); diff --git a/app/workers/builds.php b/app/workers/builds.php index 478c5c5423..ed1c16e8bf 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -208,8 +208,8 @@ class BuildsV1 extends Worker $schedule ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - - + + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); } catch (\Throwable $th) { $endTime = DateTime::now(); From 43adf758bd88f9a857e221f6d11bd98c55f4a8cb Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 12:30:57 +0100 Subject: [PATCH 60/65] Fix lockfile after merge --- composer.lock | 65 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/composer.lock b/composer.lock index 2e39916a0c..517065af56 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "bf3e2ed6ee8e49ab74af97b368b89a63", + "content-hash": "a673091aa6bd8ef01380b63245427c93", "packages": [ { "name": "adhocore/jwt", @@ -2357,6 +2357,67 @@ }, "time": "2020-10-24T07:04:59+00:00" }, + { + "name": "utopia-php/queue", + "version": "0.4.1", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/queue.git", + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.14.*", + "utopia-php/framework": "0.*.*" + }, + "require-dev": { + "laravel/pint": "^0.2.3", + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.5.5", + "swoole/ide-helper": "4.8.8", + "workerman/workerman": "^4.0" + }, + "suggest": { + "ext-swoole": "Needed to support Swoole.", + "workerman/workerman": "Needed to support Workerman." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Queue\\": "src/Queue" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A powerful task queue.", + "keywords": [ + "Tasks", + "framework", + "php", + "queue", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/queue/issues", + "source": "https://github.com/utopia-php/queue/tree/0.4.1" + }, + "time": "2022-11-15T16:56:37+00:00" + }, { "name": "utopia-php/registry", "version": "0.5.0", @@ -5241,5 +5302,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } From fdbb5b9810351e37279fce85778849847b9d8a4a Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 13:19:29 +0100 Subject: [PATCH 61/65] QA bug fixing --- app/workers/builds.php | 6 +- app/workers/functions.php | 95 +++++++++++------------- docker-compose.yml | 14 +++- src/Appwrite/Platform/Tasks/Schedule.php | 13 ++-- 4 files changed, 69 insertions(+), 59 deletions(-) diff --git a/app/workers/builds.php b/app/workers/builds.php index ed1c16e8bf..48d430eaa7 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -5,7 +5,6 @@ use Appwrite\Event\Func; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Resque\Worker; use Appwrite\Utopia\Response\Model\Deployment; -use Cron\CronExpression; use Executor\Executor; use Appwrite\Usage\Stats; use Utopia\Database\DateTime; @@ -59,6 +58,8 @@ class BuildsV1 extends Worker protected function buildDeployment(Document $project, Document $function, Document $deployment) { + global $register; + $dbForProject = $this->getProjectDB($project); $function = $dbForProject->getDocument('functions', $function->getId()); @@ -122,7 +123,6 @@ class BuildsV1 extends Worker ->trigger(); /** Trigger Functions */ - global $register; $pools = $register->get('pools'); $connection = $pools->get('queue')->pop(); $functions = new Func($connection->getResource()); @@ -202,6 +202,7 @@ class BuildsV1 extends Worker } /** Update function schedule */ + $dbForConsole = $this->getConsoleDB(); $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); @@ -240,7 +241,6 @@ class BuildsV1 extends Worker ); /** Update usage stats */ - global $register; if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $statsd = $register->get('statsd'); $usage = new Stats($statsd); diff --git a/app/workers/functions.php b/app/workers/functions.php index f1e472b471..906ae25d38 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -270,7 +270,6 @@ $server->job() throw new Exception('Missing payload'); } - var_dump(json_encode($payload)); $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; $data = $payload['data'] ?? ''; @@ -283,53 +282,6 @@ $server->job() return; } - /** - * Handle Event execution. - */ - if (!empty($events)) { - $limit = 30; - $sum = 30; - $offset = 0; - $functions = []; - /** @var Document[] $functions */ - while ($sum >= $limit) { - $functions = $dbForProject->find('functions', [ - Query::limit($limit), - Query::offset($offset), - Query::orderAsc('name'), - ]); - - $sum = \count($functions); - $offset = $offset + $limit; - - Console::log('Fetched ' . $sum . ' functions...'); - - foreach ($functions as $function) { - if (!array_intersect($events, $function->getAttribute('events', []))) { - continue; - } - Console::success('Iterating function: ' . $function->getAttribute('name')); - $execute( - statsd: $statsd, - dbForProject: $dbForProject, - project: $project, - function: $function, - queueForFunctions: $queueForFunctions, - trigger: 'event', - event: $events[0], - eventData: $eventData, - user: $user, - data: null, - executionId: null, - jwt: null - ); - Console::success('Triggered function: ' . $events[0]); - } - } - - return; - } - /** * Handle Schedule and HTTP execution. */ @@ -343,7 +295,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: 'http', + trigger: $type, executionId: $execution->getId(), event: null, eventData: null, @@ -359,7 +311,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: 'http', + trigger: $type, executionId: null, event: null, eventData: null, @@ -369,6 +321,49 @@ $server->job() statsd: $statsd, ); break; + case 'event': + if (!empty($events)) { + $limit = 30; + $sum = 30; + $offset = 0; + $functions = []; + /** @var Document[] $functions */ + while ($sum >= $limit) { + $functions = $dbForProject->find('functions', [ + Query::limit($limit), + Query::offset($offset), + Query::orderAsc('name'), + ]); + + $sum = \count($functions); + $offset = $offset + $limit; + + Console::log('Fetched ' . $sum . ' functions...'); + + foreach ($functions as $function) { + if (!array_intersect($events, $function->getAttribute('events', []))) { + continue; + } + Console::success('Iterating function: ' . $function->getAttribute('name')); + $execute( + statsd: $statsd, + dbForProject: $dbForProject, + project: $project, + function: $function, + queueForFunctions: $queueForFunctions, + trigger: $type, + event: $events[0], + eventData: $eventData, + user: $user, + data: null, + executionId: null, + jwt: null + ); + Console::success('Triggered function: ' . $events[0]); + } + } + } + break; } }); diff --git a/docker-compose.yml b/docker-compose.yml index 7b69254738..d7fe33d536 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -258,6 +258,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -291,6 +292,7 @@ services: - request-catcher environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -321,6 +323,7 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -376,6 +379,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -408,6 +412,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -444,6 +449,7 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET @@ -480,6 +486,7 @@ services: - openruntimes-executor environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -500,7 +507,6 @@ services: - _APP_USAGE_STATS - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD - - _APP_WORKER_PER_CORE appwrite-worker-mails: entrypoint: worker-mails @@ -518,6 +524,7 @@ services: # - smtp environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS @@ -548,6 +555,7 @@ services: - redis environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -572,6 +580,7 @@ services: - redis environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 @@ -624,6 +633,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -662,6 +672,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -697,6 +708,7 @@ services: - redis environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index dc86f9293e..bafedd27cb 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -97,7 +97,7 @@ class Schedule extends Action $pools->reclaim(); - Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); + Console::success("{$total} functions were loaded in " . (microtime(true) - $loadStart) . " seconds"); Console::success("Starting timers at " . DateTime::now()); @@ -152,7 +152,7 @@ class Schedule extends Action $pools->reclaim(); - Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); + Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds"); }); /** @@ -185,11 +185,14 @@ class Schedule extends Action $total++; - $promiseStart = \microtime(true); // in seconds + $promiseStart = \time(); // in seconds $executionStart = $nextDate->getTimestamp(); // in seconds $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued + $delay = $executionSleep; - $delay = \ceil(\intval($executionSleep)); + \var_dump($delay); + \var_dump(\time()); + \var_dump('---'); if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; @@ -228,7 +231,7 @@ class Schedule extends Action $timerEnd = \microtime(true); $lastEnqueueUpdate = $timerStart; - Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); + Console::log("Enqueue tick: {$total} executions were enqueued in " . ($timerEnd - $timerStart) . " seconds"); }; Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); From 3e325afb616b57ce688b08514f6000a289bdb9a0 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 13:44:14 +0100 Subject: [PATCH 62/65] Revert unnessessary changes --- app/controllers/api/functions.php | 3 +- app/workers/functions.php | 85 +++++++++++++++---------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 86981b0071..a94cde197d 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1155,7 +1155,8 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user); + ->setUser($user) + ->trigger(); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/workers/functions.php b/app/workers/functions.php index 906ae25d38..d503fdc5e0 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -282,6 +282,48 @@ $server->job() return; } + if (!empty($events)) { + $limit = 30; + $sum = 30; + $offset = 0; + $functions = []; + /** @var Document[] $functions */ + while ($sum >= $limit) { + $functions = $dbForProject->find('functions', [ + Query::limit($limit), + Query::offset($offset), + Query::orderAsc('name'), + ]); + + $sum = \count($functions); + $offset = $offset + $limit; + + Console::log('Fetched ' . $sum . ' functions...'); + + foreach ($functions as $function) { + if (!array_intersect($events, $function->getAttribute('events', []))) { + continue; + } + Console::success('Iterating function: ' . $function->getAttribute('name')); + $execute( + statsd: $statsd, + dbForProject: $dbForProject, + project: $project, + function: $function, + queueForFunctions: $queueForFunctions, + trigger: $type, + event: $events[0], + eventData: $eventData, + user: $user, + data: null, + executionId: null, + jwt: null + ); + Console::success('Triggered function: ' . $events[0]); + } + } + } + /** * Handle Schedule and HTTP execution. */ @@ -321,49 +363,6 @@ $server->job() statsd: $statsd, ); break; - case 'event': - if (!empty($events)) { - $limit = 30; - $sum = 30; - $offset = 0; - $functions = []; - /** @var Document[] $functions */ - while ($sum >= $limit) { - $functions = $dbForProject->find('functions', [ - Query::limit($limit), - Query::offset($offset), - Query::orderAsc('name'), - ]); - - $sum = \count($functions); - $offset = $offset + $limit; - - Console::log('Fetched ' . $sum . ' functions...'); - - foreach ($functions as $function) { - if (!array_intersect($events, $function->getAttribute('events', []))) { - continue; - } - Console::success('Iterating function: ' . $function->getAttribute('name')); - $execute( - statsd: $statsd, - dbForProject: $dbForProject, - project: $project, - function: $function, - queueForFunctions: $queueForFunctions, - trigger: $type, - event: $events[0], - eventData: $eventData, - user: $user, - data: null, - executionId: null, - jwt: null - ); - Console::success('Triggered function: ' . $events[0]); - } - } - } - break; } }); From 3b6174b5a7bdba2a5ddeb6c9235af2fd4dbf5c80 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 14:11:00 +0100 Subject: [PATCH 63/65] Bug fixes --- app/workers/functions.php | 47 +++++---------------------------------- 1 file changed, 6 insertions(+), 41 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index d503fdc5e0..209c9a31df 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -40,7 +40,6 @@ Server::setResource('execute', function () { string $jwt = null, Client $statsd ) { - $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); @@ -93,48 +92,13 @@ Server::setResource('execute', function () { 'search' => implode(' ', [$functionId, $executionId]), ])); - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); - } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready'); - } - - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); + // TODO: @Meldiron Trigger executions.create event here if ($execution->isEmpty()) { throw new Exception('Failed to create or read execution'); } } + $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); @@ -311,7 +275,7 @@ $server->job() project: $project, function: $function, queueForFunctions: $queueForFunctions, - trigger: $type, + trigger: 'event', event: $events[0], eventData: $eventData, user: $user, @@ -322,6 +286,7 @@ $server->job() Console::success('Triggered function: ' . $events[0]); } } + return; } /** @@ -337,7 +302,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: $type, + trigger: 'http', executionId: $execution->getId(), event: null, eventData: null, @@ -353,7 +318,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: $type, + trigger: 'schedule', executionId: null, event: null, eventData: null, From e37f70246e4865434eefc6355476a89433a02d21 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 14:34:11 +0100 Subject: [PATCH 64/65] PR review changes --- app/init.php | 1 - app/worker.php | 9 +++------ app/workers/builds.php | 10 ++-------- src/Appwrite/Platform/Tasks/Schedule.php | 7 +------ 4 files changed, 6 insertions(+), 21 deletions(-) diff --git a/app/init.php b/app/init.php index 067fd70204..31f82740b6 100644 --- a/app/init.php +++ b/app/init.php @@ -565,7 +565,6 @@ $register->set('pools', function () { $dsns = $connection['dsns'] ?? ''; $multipe = $connection['multiple'] ?? false; $schemes = $connection['schemes'] ?? []; - $useResource = $connection['useResource'] ?? true; $config = []; $dsns = explode(',', $connection['dsns'] ?? ''); diff --git a/app/worker.php b/app/worker.php index eb7e570007..42a5f92439 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,6 +16,7 @@ use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Logger\Log; +use Utopia\Logger\Logger; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); @@ -107,9 +108,7 @@ $server ->error() ->inject('error') ->inject('logger') - ->inject('register') - ->action(function ($error, $logger, $register) { - + ->action(function (Throwable $error, Logger $logger) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); if ($error instanceof PDOException) { @@ -124,7 +123,7 @@ $server $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); - $log->setAction('appwrite-worker-functions'); + $log->setAction('appwrite-queue-' . App::getEnv('QUEUE')); $log->addTag('verboseType', get_class($error)); $log->addTag('code', $error->getCode()); $log->addExtra('file', $error->getFile()); @@ -143,6 +142,4 @@ $server Console::error('[Error] Message: ' . $error->getMessage()); Console::error('[Error] File: ' . $error->getFile()); Console::error('[Error] Line: ' . $error->getLine()); - - $register->get('pools')->reclaim(); }); diff --git a/app/workers/builds.php b/app/workers/builds.php index 48d430eaa7..1fa86b99a8 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -111,15 +111,13 @@ class BuildsV1 extends Worker /** Trigger Webhook */ $deploymentModel = new Deployment(); - $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); - $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); $deploymentUpdate ->setProject($project) ->setEvent('functions.[functionId].deployments.[deploymentId].update') ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) - ->setPayload($data) + ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) ->trigger(); /** Trigger Functions */ @@ -127,11 +125,7 @@ class BuildsV1 extends Worker $connection = $pools->get('queue')->pop(); $functions = new Func($connection->getResource()); $functions - ->setData(\json_encode($data)) - ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()) + ->from($deploymentUpdate) ->trigger(); $connection->reclaim(); diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index bafedd27cb..1d89d85a04 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -187,12 +187,7 @@ class Schedule extends Action $promiseStart = \time(); // in seconds $executionStart = $nextDate->getTimestamp(); // in seconds - $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued - $delay = $executionSleep; - - \var_dump($delay); - \var_dump(\time()); - \var_dump('---'); + $delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; From 5f8d3ff6ee8a4645bc1613b144b5f1cb45badd1f Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 14:50:12 +0100 Subject: [PATCH 65/65] More PR reviews --- app/workers/functions.php | 25 ++++++++++--------------- src/Appwrite/Event/Func.php | 2 +- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 209c9a31df..3047bf027e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -27,18 +27,18 @@ Authorization::setDefaultStatus(false); Server::setResource('execute', function () { return function ( + Func $queueForFunctions, + Database $dbForProject, + Client $statsd, Document $project, Document $function, - Database $dbForProject, - Func $queueForFunctions, string $trigger, - string $executionId = null, - string $event = null, - string $eventData = null, string $data = null, ?Document $user = null, string $jwt = null, - Client $statsd + string $event = null, + string $eventData = null, + string $executionId = null, ) { $user ??= new Document(); $functionId = $function->getId(); @@ -124,11 +124,11 @@ Server::setResource('execute', function () { ]); /** Execute function */ - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { - $executionResponse = $executor->createExecution( + $client = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + $executionResponse = $client->createExecution( projectId: $project->getId(), - deploymentId: $deployment->getId(), + deploymentId: $deploymentId, payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', variables: $vars, timeout: $function->getAttribute('timeout', 0), @@ -171,12 +171,7 @@ Server::setResource('execute', function () { /** Trigger Functions */ $queueForFunctions - ->setData($data ?? '') - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) + ->from($executionUpdate) ->trigger(); /** Trigger realtime event */ diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index b114ca4b59..22940ad08e 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -156,7 +156,7 @@ class Func extends Event 'execution' => $this->execution, 'type' => $this->type, 'jwt' => $this->jwt, - 'payload' => '', + 'payload' => $this->payload, 'events' => $events, 'data' => $this->data, ]);