From 1761b77d0f50703920407d53ab4e001aece50b0e Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 9 Nov 2022 19:01:43 +0200 Subject: [PATCH 01/31] function worker --- app/controllers/api/functions.php | 37 +- app/init.php | 49 +- app/tasks/schedule.php | 20 +- app/worker.php | 64 ++ app/workers/functions.php | 612 +++++++++--------- composer.json | 3 +- composer.lock | 213 ++++-- .../Utopia/Response/Model/Execution.php | 1 + src/Appwrite/Utopia/Response/Model/Func.php | 12 - 9 files changed, 625 insertions(+), 386 deletions(-) create mode 100644 app/worker.php diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 46296bee37..5b969e63d8 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -5,7 +5,6 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\Func; use Appwrite\Event\Validator\Event as ValidatorEvent; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\CustomId; @@ -14,6 +13,7 @@ use Utopia\Database\Permission; use Utopia\Database\Role; use Utopia\Database\Validator\UID; use Appwrite\Usage\Stats; +use Utopia\Pools\Group; use Utopia\Storage\Device; use Utopia\Storage\Validator\File; use Utopia\Storage\Validator\FileExt; @@ -41,6 +41,7 @@ use Utopia\CLI\Console; use Utopia\Database\Validator\Roles; use Utopia\Validator\Boolean; use Utopia\Database\Exception\Duplicate as DuplicateException; +use Utopia\Queue\Client as queue; include_once __DIR__ . '/../shared/api.php'; @@ -1040,8 +1041,6 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') $response->noContent(); }); - - App::post('/v1/functions/:functionId/executions') ->groups(['api', 'functions']) ->desc('Create Execution') @@ -1067,7 +1066,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('events') ->inject('usage') ->inject('mode') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode) { + ->inject('pools') + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1155,17 +1155,18 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $event = new Func(); - $event - ->setType('http') - ->setExecution($execution) - ->setFunction($function) - ->setData($data) - ->setJWT($jwt) - ->setProject($project) - ->setUser($user); - - $event->trigger(); + $queue = new queue(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); + $queue->enqueue([ + 'type' => 'http', + 'value' => [ + 'type' => 'http', + 'execution' => $execution, + 'function' => $function, + 'data' => $data, + 'jwt' => $jwt, + 'project' => $project, + 'user' => $user + ]]); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1198,11 +1199,11 @@ App::post('/v1/functions/:functionId/executions') deploymentId: $deployment->getId(), path: $build->getAttribute('outputPath', ''), vars: $vars, - data: $data, entrypoint: $deployment->getAttribute('entrypoint', ''), + data: $data, runtime: $function->getAttribute('runtime', ''), - timeout: $function->getAttribute('timeout', 0), - baseImage: $runtime['image'] + baseImage: $runtime['image'], + timeout: $function->getAttribute('timeout', 0) ); /** Update execution status */ diff --git a/app/init.php b/app/init.php index a08dcfd137..4500526583 100644 --- a/app/init.php +++ b/app/init.php @@ -38,6 +38,8 @@ use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; use Appwrite\URL\URL as AppwriteURL; +use Utopia\Queue\Client as SyncOut; +use Utopia\Queue\Connection\Redis as QueueRedis; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; @@ -75,6 +77,7 @@ use Ahc\Jwt\JWTException; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; +use Utopia\Queue; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -497,6 +500,7 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); + $register->set('pools', function () { $group = new Group(); @@ -522,30 +526,35 @@ $register->set('pools', function () { 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], + 'useResource' => true, ], 'database' => [ 'type' => 'database', 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], + 'useResource' => true, ], 'queue' => [ 'type' => 'queue', 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], + 'useResource' => false, ], 'pubsub' => [ 'type' => 'pubsub', 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], + 'useResource' => true, ], 'cache' => [ 'type' => 'cache', 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), 'multiple' => true, 'schemes' => ['redis'], + 'useResource' => true, ], ]; @@ -554,6 +563,7 @@ $register->set('pools', function () { $dsns = $connection['dsns'] ?? ''; $multipe = $connection['multiple'] ?? false; $schemes = $connection['schemes'] ?? []; + $useResource = $connection['useResource'] ?? true; $config = []; $dsns = explode(',', $connection['dsns'] ?? ''); @@ -576,7 +586,7 @@ $register->set('pools', function () { $dsnScheme = $dsn->getScheme(); $dsnDatabase = $dsn->getDatabase(); - if (!in_array($dsnScheme, $schemes)) { + if (!in_array($dsnScheme, $schemes) && $useResource) { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); } @@ -635,13 +645,15 @@ $register->set('pools', function () { }; $adapter->setDefaultDatabase($dsn->getDatabase()); - - break; - case 'queue': - $adapter = $resource(); break; case 'pubsub': + break; $adapter = $resource(); + case 'queue': + $adapter = match ($dsn->getScheme()) { + 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), + default => 'bla' + }; break; case 'cache': $adapter = match ($dsn->getScheme()) { @@ -664,12 +676,6 @@ $register->set('pools', function () { Config::setParam('pools-' . $key, $config); } - try { - $group->fill(); - } catch (\Throwable $th) { - Console::error('Connection failure: ' . $th->getMessage()); - } - return $group; }); $register->set('influxdb', function () { @@ -847,8 +853,7 @@ App::setResource('messaging', fn() => new Phone()); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); - -App::setResource('clients', function ($request, $console, $project) { +App::setResource('clients', function ($request, $console, $project) use ($register) { $console->setAttribute('platforms', [ // Always allow current host '$collection' => ID::custom('platforms'), 'name' => 'Current Host', @@ -1024,6 +1029,24 @@ App::setResource('console', function () { ]); }, []); +App::setResource('queue', function () { + + $fallbackForRedis = AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); + + $connection = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); + + $dsns = explode(',', $connection ?? ''); + $dsn = explode('=', $dsns[0]); + $dsn = $dsn[1] ?? ''; + return new DSN($dsn); +}, []); + App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index a478267b8f..f370d7da41 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -3,12 +3,14 @@ global $cli; global $register; +use Appwrite\Event\Event; use Cron\CronExpression; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\DateTime; use Utopia\Database\Query; use Swoole\Timer; +use Utopia\Queue\Client as worker; const FUNCTION_UPDATE_TIMER = 60; //seconds const FUNCTION_ENQUEUE_TIMER = 60; //seconds @@ -108,7 +110,7 @@ $cli * The timer updates $functions from db on last resourceUpdatedAt attr in X-min. */ Co\run( - function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { + function () use ($register, $removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { Timer::tick(FUNCTION_UPDATE_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) { $time = DateTime::now(); $limit = 1000; @@ -162,7 +164,7 @@ $cli /** * The timer sends to worker every 1 min and re-enqueue matched functions. */ - Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) { + Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($register, $dbForConsole, &$functions, &$queue) { $timerStart = \microtime(true); $time = DateTime::now(); $timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */ @@ -175,9 +177,17 @@ $cli console::info(count($schedule) . " functions sent to worker for time slot " . $slot); foreach ($schedule as $function) { - /** - * Enqueue function (here should be the Enqueue call - */ + $pools = $register->get('pools'); + $worker = new worker(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); + $project = $dbForConsole->getDocument('projects', $function['projectId']); + $worker + ->enqueue([ + 'type' => 'schedule', + 'value' => [ + 'project' => $project, + 'function' => getProjectDB($project)->getDocument('functions', $function['projectId']), + ] + ]); //Console::warning("Enqueueing :{$function['resourceId']}"); $cron = new CronExpression($function['schedule']); $next = DateTime::format($cron->getNextRunDate()); diff --git a/app/worker.php b/app/worker.php new file mode 100644 index 0000000000..eaef650d99 --- /dev/null +++ b/app/worker.php @@ -0,0 +1,64 @@ + $register); + +Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { + $pools = $register->get('pools'); + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('console'); + + return $database; +}, ['cache', 'register']); + +Server::setResource('cache', function (Registry $register) { + $pools = $register->get('pools'); + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['register']); + +App::setResource('logger', function ($register) { + return $register->get('logger'); +}, ['register']); + + +$pools = $register->get('pools'); +$client = $pools->get('queue')->pop()->getResource(); + + +$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); +$workerNumber = 1; + +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); diff --git a/app/workers/functions.php b/app/workers/functions.php index 1ba4b6575b..47d36e4ce2 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -1,100 +1,315 @@ getId(); + $deploymentId = $function->getAttribute('deployment', ''); + + /** Check if deployment exists */ + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if ($deployment->getAttribute('resourceId') !== $functionId) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); } - public function init(): void - { - $this->executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); } - public function run(): void - { - $type = $this->args['type'] ?? ''; - $events = $this->args['events'] ?? []; - $project = new Document($this->args['project'] ?? []); - $user = new Document($this->args['user'] ?? []); - $payload = json_encode($this->args['payload'] ?? []); + /** Check if build has exists */ + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + if ($build->isEmpty()) { + throw new Exception('Build not found', 404); + } + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready', 400); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready', 400); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $carry[$var->getAttribute('key')] = $var->getAttribute('value'); + return $carry; + }, []); + + /** Collect environment variables */ + $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_ID' => $functionId, + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, + 'APPWRITE_FUNCTION_TRIGGER' => $trigger, + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_FUNCTION_EVENT' => $event ?? '', + 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', + 'APPWRITE_FUNCTION_DATA' => $data ?? '', + 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', + 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', + ]); + + /** Execute function */ + try { + $executionResponse = $executor->createExecution( + projectId: $project->getId(), + deploymentId: $deploymentId, + path: $build->getAttribute('outputPath', ''), + vars: $vars, + entrypoint: $deployment->getAttribute('entrypoint', ''), + data: $vars['APPWRITE_FUNCTION_DATA'] ?? '', + runtime: $function->getAttribute('runtime', ''), + baseImage: $runtime['image'], + timeout: $function->getAttribute('timeout', 0) + ); + + /** Update execution status */ + $execution + ->setAttribute('status', $executionResponse['status']) + ->setAttribute('statusCode', $executionResponse['statusCode']) + ->setAttribute('response', $executionResponse['response']) + ->setAttribute('stdout', $executionResponse['stdout']) + ->setAttribute('stderr', $executionResponse['stderr']) + ->setAttribute('duration', $executionResponse['duration']); + } catch (\Throwable $th) { + $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); + $execution + ->setAttribute('duration', (float)$interval->format('%s.%f')) + ->setAttribute('status', 'failed') + ->setAttribute('statusCode', $th->getCode()) + ->setAttribute('stderr', $th->getMessage()); + Console::error($th->getMessage()); + } + + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + /** Trigger Webhook */ + $executionModel = new Execution(); + $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); + $executionUpdate + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) + ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) + ->trigger(); + + /** Trigger Functions */ + $executionUpdate + ->setClass(Event::FUNCTIONS_CLASS_NAME) + ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + ->trigger(); + + /** Trigger realtime event */ + $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ + 'functionId' => $function->getId(), + 'executionId' => $execution->getId() + ]); + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $execution + ); + Realtime::send( + projectId: 'console', + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + Realtime::send( + projectId: $project->getId(), + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + /** Update usage stats */ + global $register; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $statsd = $register->get('statsd'); + $usage = new Stats($statsd); + $usage + ->setParam('projectId', $project->getId()) + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')) + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit(); + } +}; + +$adapter = new Queue\Adapter\Swoole($client, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); +$server = new Queue\Server($adapter); + +$server->job() + ->inject('message') + ->inject('dbForProject') + ->action(function (Message $message, Database $dbForProject) use ($execute) { + $args = $message->getPayload()['value'] ?? []; + $type = $message->getPayload()['type'] ?? ''; + $events = $args['events'] ?? []; + $project = new Document($args['project'] ?? []); + $user = new Document($args['user'] ?? []); + // Where $payload comes from + $payload = json_encode($args['payload'] ?? []); if ($project->getId() === 'console') { return; } - $database = $this->getProjectDB($project); - /** * Handle Event execution. */ if (!empty($events)) { $limit = 30; - $sum = 30; - $offset = 0; - $functions = []; - /** @var Document[] $functions */ + $sum = $limit; + $total = 0; + $latestDocument = null; - while ($sum >= $limit) { - $functions = $database->find('functions', [ - Query::limit($limit), - Query::offset($offset), - Query::orderAsc('name'), - ]); - $sum = \count($functions); - $offset = $offset + $limit; + while ($sum === $limit) { + $paginationQueries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $paginationQueries[] = Query::cursorAfter($latestDocument); + } + $results = $dbForProject->find('functions', \array_merge($paginationQueries, [ + Query::orderAsc('name') + ])); + + $sum = count($results); + $total = $total + $sum; Console::log('Fetched ' . $sum . ' functions...'); - foreach ($functions as $function) { + foreach ($results as $function) { if (!array_intersect($events, $function->getAttribute('events', []))) { continue; } Console::success('Iterating function: ' . $function->getAttribute('name')); - $this->execute( - project: $project, - function: $function, - dbForProject: $database, - trigger: 'event', - // Pass first, most verbose event pattern - event: $events[0], - eventData: $payload, - user: $user - ); + // As event, pass first, most verbose event pattern + call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null); Console::success('Triggered function: ' . $events[0]); } + + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } return; @@ -103,33 +318,20 @@ class FunctionsV1 extends Worker /** * Handle Schedule and HTTP execution. */ - $user = new Document($this->args['user'] ?? []); - $project = new Document($this->args['project'] ?? []); - $execution = new Document($this->args['execution'] ?? []); - $function = new Document($this->args['function'] ?? []); + $user = new Document($args['user'] ?? []); + $project = new Document($args['project'] ?? []); + $execution = new Document($args['execution'] ?? []); + $function = new Document($args['function'] ?? []); switch ($type) { case 'http': - $jwt = $this->args['jwt'] ?? ''; - $data = $this->args['data'] ?? ''; - - $function = $database->getDocument('functions', $execution->getAttribute('functionId')); - - $this->execute( - project: $project, - function: $function, - dbForProject: $database, - executionId: $execution->getId(), - trigger: 'http', - data: $data, - user: $user, - jwt: $jwt - ); - + $jwt = $args['jwt'] ?? ''; + $data = $args['data'] ?? ''; + $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); + call_user_func($execute, $project, $function, $dbForProject, 'http', $execution->getId(), null, null, $data, $user, $jwt); break; case 'schedule': - $functionOriginal = $function; /* * 1. Get Original Task * 2. Check for updates @@ -143,242 +345,54 @@ class FunctionsV1 extends Worker * If error count bigger than allowed change status to pause */ - // Reschedule - $function = $database->getDocument('functions', $function->getId()); - - if (empty($function->getId())) { - throw new Exception('Function not found (' . $function->getId() . ')'); - } - - if ($functionOriginal->getAttribute('schedule') !== $function->getAttribute('schedule')) { // Schedule has changed from previous run, ignore this run. - return; - } - - if ($functionOriginal->getAttribute('scheduleUpdatedAt') !== $function->getAttribute('scheduleUpdatedAt')) { // Double execution due to rapid cron changes, ignore this run. - return; - } - - $cron = new CronExpression($function->getAttribute('schedule')); - $next = DateTime::format($cron->getNextRunDate()); - - $function = $function - ->setAttribute('scheduleNext', $next) - ->setAttribute('schedulePrevious', DateTime::now()); - - $function = $database->updateDocument( - 'functions', - $function->getId(), - $function - ); - - $reschedule = new Func(); - $reschedule - ->setFunction($function) - ->setType('schedule') - ->setUser($user) - ->setProject($project) - ->schedule(new \DateTime($next)); - ; - - $this->execute( - project: $project, - function: $function, - dbForProject: $database, - trigger: 'schedule' - ); - + call_user_func($execute, $project, $function, $dbForProject, 'schedule', null, null, null, null, null, null); break; } - } + }); - private function execute( - Document $project, - Document $function, - Database $dbForProject, - string $trigger, - string $executionId = null, - string $event = null, - string $eventData = null, - string $data = null, - ?Document $user = null, - string $jwt = null - ) { +$server + ->error() + ->inject('error') + ->inject('logger') + ->inject('register') + ->action(function ($error, $logger, $register) { - $user ??= new Document(); - $functionId = $function->getId(); - $deploymentId = $function->getAttribute('deployment', ''); + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - /** Check if deployment exists */ - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - if ($deployment->getAttribute('resourceId') !== $functionId) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + if ($error instanceof PDOException) { + throw $error; } - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + if ($error->getCode() >= 500 || $error->getCode() === 0) { + $log = new Log(); + + $log->setNamespace("appwrite-worker"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + $log->setAction('appwrite-worker-functions'); + $log->addTag('verboseType', get_class($error)); + $log->addTag('code', $error->getCode()); + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + $log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $logger->addLog($log); } - /** Check if build has exists */ - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - if ($build->isEmpty()) { - throw new Exception('Build not found', 404); - } + Console::error('[Error] Type: ' . get_class($error)); + Console::error('[Error] Message: ' . $error->getMessage()); + Console::error('[Error] File: ' . $error->getFile()); + Console::error('[Error] Line: ' . $error->getLine()); - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready', 400); - } + $register->get('pools')->reclaim(); + }); - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); - - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); - } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { - $carry[$var->getAttribute('key')] = $var->getAttribute('value'); - return $carry; - }, []); - - /** Collect environment variables */ - $vars = \array_merge($vars, [ - 'APPWRITE_FUNCTION_ID' => $functionId, - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, - 'APPWRITE_FUNCTION_TRIGGER' => $trigger, - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_EVENT' => $event ?? '', - 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', - 'APPWRITE_FUNCTION_DATA' => $data ?? '', - 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', - 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', - ]); - - /** Execute function */ - try { - $executionResponse = $this->executor->createExecution( - projectId: $project->getId(), - deploymentId: $deploymentId, - path: $build->getAttribute('outputPath', ''), - vars: $vars, - entrypoint: $deployment->getAttribute('entrypoint', ''), - data: $vars['APPWRITE_FUNCTION_DATA'] ?? '', - runtime: $function->getAttribute('runtime', ''), - timeout: $function->getAttribute('timeout', 0), - baseImage: $runtime['image'] - ); - - /** Update execution status */ - $execution - ->setAttribute('status', $executionResponse['status']) - ->setAttribute('statusCode', $executionResponse['statusCode']) - ->setAttribute('response', $executionResponse['response']) - ->setAttribute('stdout', $executionResponse['stdout']) - ->setAttribute('stderr', $executionResponse['stderr']) - ->setAttribute('duration', $executionResponse['duration']); - } catch (\Throwable $th) { - $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); - $execution - ->setAttribute('duration', (float)$interval->format('%s.%f')) - ->setAttribute('status', 'failed') - ->setAttribute('statusCode', $th->getCode()) - ->setAttribute('stderr', $th->getMessage()); - Console::error($th->getMessage()); - } - - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - /** Trigger Webhook */ - $executionModel = new Execution(); - $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); - $executionUpdate - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) - ->trigger(); - - /** Trigger Functions */ - $executionUpdate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) - ->trigger(); - - /** Trigger realtime event */ - $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ - 'functionId' => $function->getId(), - 'executionId' => $execution->getId() - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $execution - ); - Realtime::send( - projectId: 'console', - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - /** Update usage stats */ - global $register; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $statsd = $register->get('statsd'); - $usage = new Stats($statsd); - $usage - ->setParam('projectId', $project->getId()) - ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')) - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ->submit(); - } - } - - public function shutdown(): void - { - } -} +$server->workerStart(); +$server->start(); diff --git a/composer.json b/composer.json index e09bec422e..6843ef6894 100644 --- a/composer.json +++ b/composer.json @@ -61,7 +61,8 @@ "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.6.*", - "utopia-php/pools": "0.1.*", + "utopia-php/queue": "0.4.0", + "utopia-php/pools": "dev-upgrade-cli as 0.2.0", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index 28d48b7f3d..6c6d129458 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f3beee3a829a19e53b311052111bde2c", + "content-hash": "a744959294e219fff6ea9c17f9fb0705", "packages": [ { "name": "adhocore/jwt", @@ -115,15 +115,15 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.11.0", + "version": "0.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "547fc026e11c0946846a8ac690898f5bf53be101" + "reference": "9d74a477ba3333cbcfac565c46fcf19606b7b603" }, "require": { "php": ">=8.0", - "utopia-php/system": "0.4.*" + "utopia-php/system": "0.6.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -154,7 +154,7 @@ "php", "runtimes" ], - "time": "2022-08-15T14:03:36+00:00" + "time": "2022-11-07T16:45:52+00:00" }, { "name": "chillerlan/php-qrcode", @@ -300,16 +300,16 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.13.1", + "version": "v1.14.0", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49" + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/dccc8a46586475075fbb012d8bd523b8a938c2dc", + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc", "shasum": "" }, "require": { @@ -341,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1" + "source": "https://github.com/colinmollenhour/credis/tree/v1.14.0" }, - "time": "2022-06-20T22:56:59+00:00" + "time": "2022-11-09T01:18:39+00:00" }, { "name": "composer/package-versions-deprecated", @@ -693,16 +693,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.1", + "version": "2.4.3", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379" + "reference": "67c26b443f348a51926030c83481b85718457d3d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/69568e4293f4fa993f3b0e51c9723e1e17c41379", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", + "reference": "67c26b443f348a51926030c83481b85718457d3d", "shasum": "" }, "require": { @@ -792,7 +792,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.1" + "source": "https://github.com/guzzle/psr7/tree/2.4.3" }, "funding": [ { @@ -808,7 +808,7 @@ "type": "tidelift" } ], - "time": "2022-08-28T14:45:39+00:00" + "time": "2022-10-26T14:07:24+00:00" }, { "name": "influxdb/influxdb-php", @@ -931,6 +931,72 @@ }, "time": "2021-02-04T16:20:16+00:00" }, + { + "name": "laravel/pint", + "version": "v1.2.0", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.11.0", + "illuminate/view": "^9.27", + "laravel-zero/framework": "^9.1.3", + "mockery/mockery": "^1.5.0", + "nunomaduro/larastan": "^2.2", + "nunomaduro/termwind": "^1.14.0", + "pestphp/pest": "^1.22.1" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2022-09-13T15:07:15+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -2431,23 +2497,24 @@ }, { "name": "utopia-php/pools", - "version": "0.1.0", + "version": "dev-upgrade-cli", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805" + "reference": "88a2c1ed2badbfdf2787ce0a12def2c988fc1097" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/5a467a569a80aefc846a97dc195b4adc2fd71805", - "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/88a2c1ed2badbfdf2787ce0a12def2c988fc1097", + "reference": "88a2c1ed2badbfdf2787ce0a12def2c988fc1097", "shasum": "" }, "require": { "ext-mongodb": "*", "ext-pdo": "*", "ext-redis": "*", - "php": ">=8.0" + "php": ">=8.0", + "utopia-php/cli": "0.13.*" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -2478,9 +2545,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.1.0" + "source": "https://github.com/utopia-php/pools/tree/upgrade-cli" }, - "time": "2022-10-11T19:31:07+00:00" + "time": "2022-11-04T08:33:04+00:00" }, { "name": "utopia-php/preloader", @@ -2535,6 +2602,67 @@ }, "time": "2020-10-24T07:04:59+00:00" }, + { + "name": "utopia-php/queue", + "version": "0.4.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/queue.git", + "reference": "0cad4cf4231377aa6c67956b51ba1954e0d02166" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/0cad4cf4231377aa6c67956b51ba1954e0d02166", + "reference": "0cad4cf4231377aa6c67956b51ba1954e0d02166", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.13.*", + "utopia-php/framework": "0.*.*" + }, + "require-dev": { + "laravel/pint": "^0.2.3", + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.5.5", + "swoole/ide-helper": "4.8.8", + "workerman/workerman": "^4.0" + }, + "suggest": { + "ext-swoole": "Needed to support Swoole.", + "workerman/workerman": "Needed to support Workerman." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Queue\\": "src/Queue" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A powerful task queue.", + "keywords": [ + "Tasks", + "framework", + "php", + "queue", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/queue/issues", + "source": "https://github.com/utopia-php/queue/tree/0.4.0" + }, + "time": "2022-10-31T06:23:08+00:00" + }, { "name": "utopia-php/registry", "version": "0.5.0", @@ -2700,23 +2828,25 @@ }, { "name": "utopia-php/system", - "version": "0.4.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0" + "reference": "289c4327713deadc9c748b5317d248133a02f245" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0", + "url": "https://api.github.com/repos/utopia-php/system/zipball/289c4327713deadc9c748b5317d248133a02f245", + "reference": "289c4327713deadc9c748b5317d248133a02f245", "shasum": "" }, "require": { + "laravel/pint": "1.2.*", "php": ">=7.4" }, "require-dev": { "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", "vimeo/psalm": "4.0.1" }, "type": "library", @@ -2749,9 +2879,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.4.0" + "source": "https://github.com/utopia-php/system/tree/0.6.0" }, - "time": "2021-02-04T14:14:49+00:00" + "time": "2022-11-07T13:51:59+00:00" }, { "name": "utopia-php/websocket", @@ -3574,16 +3704,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.17", + "version": "9.2.18", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8" + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", "shasum": "" }, "require": { @@ -3639,7 +3769,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" }, "funding": [ { @@ -3647,7 +3777,7 @@ "type": "github" } ], - "time": "2022-08-30T12:24:04+00:00" + "time": "2022-10-27T13:35:33+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5402,11 +5532,18 @@ "version": "dev-feat-update-cache-lib", "alias": "0.26.1", "alias_normalized": "0.26.1.0" + }, + { + "package": "utopia-php/pools", + "version": "dev-upgrade-cli", + "alias": "0.2.0", + "alias_normalized": "0.2.0.0" } ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/database": 20 + "utopia-php/database": 20, + "utopia-php/pools": 20 }, "prefer-stable": false, "prefer-lowest": false, @@ -5431,5 +5568,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Utopia/Response/Model/Execution.php b/src/Appwrite/Utopia/Response/Model/Execution.php index 13011a24b7..987a140dfb 100644 --- a/src/Appwrite/Utopia/Response/Model/Execution.php +++ b/src/Appwrite/Utopia/Response/Model/Execution.php @@ -6,6 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use Utopia\Database\Role; + class Execution extends Model { public function __construct() diff --git a/src/Appwrite/Utopia/Response/Model/Func.php b/src/Appwrite/Utopia/Response/Model/Func.php index c7e69fff88..540b143876 100644 --- a/src/Appwrite/Utopia/Response/Model/Func.php +++ b/src/Appwrite/Utopia/Response/Model/Func.php @@ -81,18 +81,6 @@ class Func extends Model 'default' => '', 'example' => '5 4 * * *', ]) - ->addRule('scheduleNext', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Function\'s next scheduled execution time in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) - ->addRule('schedulePrevious', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Function\'s previous scheduled execution time in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) ->addRule('timeout', [ 'type' => self::TYPE_INTEGER, 'description' => 'Function execution timeout in seconds.', From 134dba0b4354f56c214393f8df3c0385f8527b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 12 Nov 2022 14:35:42 +0000 Subject: [PATCH 02/31] Fix startup errors --- app/preload.php | 1 + app/tasks/schedule.php | 1 + app/worker.php | 25 ++++++++++++++++++++-- app/workers/builds.php | 3 +++ app/workers/functions.php | 2 +- composer.lock | 44 +++++++++++++++++++-------------------- 6 files changed, 51 insertions(+), 25 deletions(-) diff --git a/app/preload.php b/app/preload.php index 4935db3da4..e587bfaed5 100644 --- a/app/preload.php +++ b/app/preload.php @@ -36,6 +36,7 @@ foreach ( realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload + realpath(__DIR__ . '/../vendor/utopia-php/queue/src/Queue/Adapter/Workerman.php'), // TODO: remove workerman autoload ] as $key => $value ) { if ($value !== false) { diff --git a/app/tasks/schedule.php b/app/tasks/schedule.php index f370d7da41..121885b6c9 100644 --- a/app/tasks/schedule.php +++ b/app/tasks/schedule.php @@ -95,6 +95,7 @@ $cli 'resourceId' => $document->getAttribute('resourceId'), 'schedule' => $document->getAttribute('schedule'), 'resourceUpdatedAt' => $document->getAttribute('resourceUpdatedAt'), + 'projectId' => $document->getAttribute('projectId') ]; } diff --git a/app/worker.php b/app/worker.php index eaef650d99..bb35576172 100644 --- a/app/worker.php +++ b/app/worker.php @@ -9,14 +9,14 @@ use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; global $register; - - Server::setResource('register', fn() => $register); Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { @@ -33,6 +33,27 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) return $database; }, ['cache', 'register']); +Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) { + $args = $message->getPayload()['value'] ?? []; + $project = new Document($args['project'] ?? []); + + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } + + $pools = $register->get('pools'); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +}, ['cache', 'register', 'message', 'dbForConsole']); + Server::setResource('cache', function (Registry $register) { $pools = $register->get('pools'); $list = Config::getParam('pools-cache', []); diff --git a/app/workers/builds.php b/app/workers/builds.php index 235ad0da72..42831fb029 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -188,11 +188,14 @@ class BuildsV1 extends Worker } /** Update function schedule */ + // TODO: @Meldiron refactor scheduler here + /* $schedule = $function->getAttribute('schedule', ''); $cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null; $next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? DateTime::format($cron->getNextRunDate()) : null; $function->setAttribute('scheduleNext', $next); $function = $dbForProject->updateDocument('functions', $function->getId(), $function); + */ } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); diff --git a/app/workers/functions.php b/app/workers/functions.php index 47d36e4ce2..86fc2998b3 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -27,7 +27,7 @@ Authorization::setDefaultStatus(false); global $client; global $workerNumber; -$executor = new Executor(App::getEnv('_APP_FUNCTIONS_PROXY_HOST')); +$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); $execute = function ( Document $project, diff --git a/composer.lock b/composer.lock index 6c6d129458..48b4039b7f 100644 --- a/composer.lock +++ b/composer.lock @@ -1724,16 +1724,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace" + "reference": "7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/cfa0ae98841b9e461207c13ab093d76b0fa7bace", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936", + "reference": "7a6ff3f1959bb01aefccb463a0f2cd3d3d2fd936", "shasum": "" }, "require": { @@ -1742,7 +1742,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -1787,7 +1787,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.27.0" }, "funding": [ { @@ -1803,7 +1803,7 @@ "type": "tidelift" } ], - "time": "2022-05-10T07:21:04+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "utopia-php/abuse", @@ -5187,16 +5187,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a", + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a", "shasum": "" }, "require": { @@ -5211,7 +5211,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -5249,7 +5249,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0" }, "funding": [ { @@ -5265,20 +5265,20 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534", + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534", "shasum": "" }, "require": { @@ -5293,7 +5293,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -5332,7 +5332,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0" }, "funding": [ { @@ -5348,7 +5348,7 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "textalk/websocket", @@ -5568,5 +5568,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From 8bcf349c3814c996abebd12740c2ffe45a71cd5a Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:03:42 +0200 Subject: [PATCH 03/31] addressing some comments --- app/controllers/api/functions.php | 10 +++++----- app/init.php | 17 +++++------------ app/worker.php | 4 +--- app/workers/functions.php | 21 +++++++++++---------- 4 files changed, 22 insertions(+), 30 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5b969e63d8..6f09e27385 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -41,7 +41,7 @@ use Utopia\CLI\Console; use Utopia\Database\Validator\Roles; use Utopia\Validator\Boolean; use Utopia\Database\Exception\Duplicate as DuplicateException; -use Utopia\Queue\Client as queue; +use Utopia\Queue\Client as QueueClient; include_once __DIR__ . '/../shared/api.php'; @@ -1155,10 +1155,10 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $queue = new queue(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); - $queue->enqueue([ - 'type' => 'http', - 'value' => [ + $queueForFunctions = new QueueClient(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); + $queueForFunctions->enqueue([ + 'type' => 'http', + 'value' => [ 'type' => 'http', 'execution' => $execution, 'function' => $function, diff --git a/app/init.php b/app/init.php index 4500526583..c5e898d836 100644 --- a/app/init.php +++ b/app/init.php @@ -38,8 +38,6 @@ use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; use Appwrite\URL\URL as AppwriteURL; -use Utopia\Queue\Client as SyncOut; -use Utopia\Queue\Connection\Redis as QueueRedis; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; @@ -67,7 +65,6 @@ use Utopia\Storage\Device\Wasabi; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; -use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\MySQL; use Utopia\Pools\Group; @@ -526,35 +523,30 @@ $register->set('pools', function () { 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], - 'useResource' => true, ], 'database' => [ 'type' => 'database', 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], - 'useResource' => true, ], 'queue' => [ 'type' => 'queue', 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], - 'useResource' => false, ], 'pubsub' => [ 'type' => 'pubsub', 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], - 'useResource' => true, ], 'cache' => [ 'type' => 'cache', 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), 'multiple' => true, 'schemes' => ['redis'], - 'useResource' => true, ], ]; @@ -586,7 +578,7 @@ $register->set('pools', function () { $dsnScheme = $dsn->getScheme(); $dsnDatabase = $dsn->getDatabase(); - if (!in_array($dsnScheme, $schemes) && $useResource) { + if (!in_array($dsnScheme, $schemes)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); } @@ -647,12 +639,12 @@ $register->set('pools', function () { $adapter->setDefaultDatabase($dsn->getDatabase()); break; case 'pubsub': - break; $adapter = $resource(); + break; case 'queue': $adapter = match ($dsn->getScheme()) { 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), - default => 'bla' + default => null }; break; case 'cache': @@ -678,6 +670,7 @@ $register->set('pools', function () { return $group; }); + $register->set('influxdb', function () { // Register DB connection $host = App::getEnv('_APP_INFLUXDB_HOST', ''); @@ -853,7 +846,7 @@ App::setResource('messaging', fn() => new Phone()); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); -App::setResource('clients', function ($request, $console, $project) use ($register) { +App::setResource('clients', function ($request, $console, $project) { $console->setAttribute('platforms', [ // Always allow current host '$collection' => ID::custom('platforms'), 'name' => 'Current Host', diff --git a/app/worker.php b/app/worker.php index bb35576172..4e484e1545 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,7 +2,6 @@ require_once __DIR__ . '/init.php'; - use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -14,7 +13,6 @@ use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; - global $register; Server::setResource('register', fn() => $register); @@ -76,7 +74,7 @@ App::setResource('logger', function ($register) { $pools = $register->get('pools'); -$client = $pools->get('queue')->pop()->getResource(); +$connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); diff --git a/app/workers/functions.php b/app/workers/functions.php index 86fc2998b3..91edee19fe 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -20,11 +20,12 @@ use Utopia\Database\Query; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; +use Utopia\Queue\Client as QueueClient; Authorization::disable(); Authorization::setDefaultStatus(false); -global $client; +global $connection; global $workerNumber; $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -40,7 +41,7 @@ $execute = function ( string $data = null, ?Document $user = null, string $jwt = null -) use ($executor) { +) use ($executor, $register) { $user ??= new Document(); $functionId = $function->getId(); @@ -50,28 +51,28 @@ $execute = function ( $deployment = $dbForProject->getDocument('deployments', $deploymentId); if ($deployment->getAttribute('resourceId') !== $functionId) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } if ($deployment->isEmpty()) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404); + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } /** Check if build has exists */ $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($build->isEmpty()) { - throw new Exception('Build not found', 404); + throw new Exception('Build not found'); } if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready', 400); + throw new Exception('Build not ready'); } /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } $runtime = $runtimes[$function->getAttribute('runtime')]; @@ -102,14 +103,14 @@ $execute = function ( $execution = $dbForProject->updateDocument('executions', $executionId, $execution); if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready', 400); + throw new Exception('Build not ready'); } /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400); + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } $runtime = $runtimes[$function->getAttribute('runtime')]; @@ -254,7 +255,7 @@ $execute = function ( } }; -$adapter = new Queue\Adapter\Swoole($client, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); +$adapter = new Queue\Adapter\Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); $server = new Queue\Server($adapter); $server->job() From 3682c1d05bbfe6a5c18355655936aa3da53c30ae Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:07:13 +0200 Subject: [PATCH 04/31] addressing some comments --- bin/worker-functions | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/bin/worker-functions b/bin/worker-functions index 5e9728c46e..7e7b4e565f 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=0.1 QUEUE='v1-functions' APP_INCLUDE='/usr/src/code/app/workers/functions.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/cli.php worker-functions $@ \ No newline at end of file From 0d79f19dcd2c4f4f22e1d5e8336ff9965fabecc4 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:14:52 +0200 Subject: [PATCH 05/31] addressing some comments --- app/controllers/api/functions.php | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 6f09e27385..fb688aaae3 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -90,7 +90,7 @@ App::post('/v1/functions') 'search' => implode(' ', [$functionId, $name, $runtime]) ])); - $log = Authorization::skip( + $schedule = Authorization::skip( fn() => $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region 'resourceType' => 'function', @@ -102,7 +102,7 @@ App::post('/v1/functions') ])) ); - $function->setAttribute('scheduleId', $log->getId()); + $function->setAttribute('scheduleId', $schedule->getId()); $dbForProject->updateDocument('functions', $function->getId(), $function); $eventsInstance->setParam('functionId', $function->getId()); @@ -470,21 +470,21 @@ App::put('/v1/functions/:functionId') 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); /** * In case we want to clear the schedule */ if (!empty($function->getAttribute('deployment'))) { - $log->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); + $schedule->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); } - $log + $schedule ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); $eventsInstance->setParam('functionId', $function->getId()); @@ -538,18 +538,18 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') 'deployment' => $deployment->getId() ]))); - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); $active = !empty($function->getAttribute('schedule')); if ($active) { - $log->setAttribute('resourceUpdatedAt', datetime::now()); + $schedule->setAttribute('resourceUpdatedAt', datetime::now()); } - $log->setAttribute('active', $active); + $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $events @@ -591,15 +591,15 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); - $log + $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('active', false) ; - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $deletes @@ -795,18 +795,18 @@ App::post('/v1/functions/:functionId/deployments') * TODO Should we update also the function collection with the scheduleUpdatedAt attr? */ - $log = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); $active = !empty($function->getAttribute('schedule')); if ($active) { - $log->setAttribute('resourceUpdatedAt', datetime::now()); + $schedule->setAttribute('resourceUpdatedAt', datetime::now()); } - $log->setAttribute('active', $active); + $schedule->setAttribute('active', $active); - Authorization::skip(function () use ($dbForConsole, $log) { - $dbForConsole->updateDocument('schedules', $log->getId(), $log); + Authorization::skip(function () use ($dbForConsole, $schedule) { + $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); }); $metadata = null; From 11915e1667bafbd36020e3c6bd6e18323f8be17f Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:20:50 +0200 Subject: [PATCH 06/31] addressing some comments --- bin/worker-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/worker-functions b/bin/worker-functions index 7e7b4e565f..6eaaeafe00 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php worker-functions $@ \ No newline at end of file +php php /usr/src/code/app/workers/worker-functions.php $@ \ No newline at end of file From 797c3f2443a020c6d836b14fc78a812562ac6d23 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Nov 2022 18:21:54 +0200 Subject: [PATCH 07/31] addressing some comments --- bin/worker-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/worker-functions b/bin/worker-functions index 6eaaeafe00..143b431b0c 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php php /usr/src/code/app/workers/worker-functions.php $@ \ No newline at end of file +php php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file From 855c3d3b84f48621470081def0e3999c2595d650 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 17:37:16 +0100 Subject: [PATCH 08/31] Re-implement removed stuff during merge --- src/Appwrite/Platform/Tasks/Schedule.php | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index 0e48d23883..39e0686da5 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -12,6 +12,8 @@ use Utopia\Database\Query; use Swoole\Timer; use Utopia\Database\Database; use Utopia\Pools\Group; +use Utopia\Queue\Client as Worker; +use Appwrite\Event\Event; use function Swoole\Coroutine\run; @@ -158,7 +160,7 @@ class Schedule extends Action * The timer to prepare soon-to-execute schedules. */ $lastEnqueueUpdate = null; - $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate) { + $enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate, $pools) { $timerStart = \microtime(true); $time = DateTime::now(); @@ -198,7 +200,7 @@ class Schedule extends Action } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $schedules, $scheduleKeys) { + \go(function () use ($delay, $schedules, $scheduleKeys, $pools) { \sleep($delay); // in seconds foreach ($scheduleKeys as $scheduleKey) { @@ -207,7 +209,21 @@ class Schedule extends Action return; } - Console::success("Executing function at " . DateTime::now()); // TODO: Send to worker queue + $schedule = $schedules[$scheduleKey]; + + $queue = $pools->get('queue')->pop(); + + $worker = new Worker(Event::FUNCTIONS_QUEUE_NAME, $queue->getResource()); + $worker + ->enqueue([ + 'type' => 'schedule', + 'value' => [ + 'project' => $schedule['project'], + 'function' => $schedule['function'], + ] + ]); + + $queue->reclaim(); } }); } From 280a44e1cd2e3dd0e35d3be5b0ae0ee60c59c00f Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 17:55:30 +0100 Subject: [PATCH 09/31] Fix bugs during QA --- app/workers/deletes.php | 26 ++------------ app/workers/functions.php | 13 ++++--- bin/worker-functions | 2 +- composer.lock | 72 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 79 insertions(+), 34 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 0134fad66d..3e0df8ce38 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -468,19 +468,7 @@ class DeletesV1 extends Worker Query::equal('functionId', [$functionId]) ], $dbForProject); - /** - * Request executor to delete all deployment containers - * TODO: Re-enable. Disabled for now because of proxy. Container killed after inactivity automatically. - */ - // Console::info("Requesting executor to delete all deployment containers for function " . $functionId); - // $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - // foreach ($deploymentIds as $deploymentId) { - // try { - // $executor->deleteRuntime($projectId, $deploymentId); - // } catch (Throwable $th) { - // Console::error($th->getMessage()); - // } - // } + // TODO: Request executor to delete runtime } /** @@ -520,17 +508,7 @@ class DeletesV1 extends Worker } }); - /** - * Request executor to delete the deployment container. - * TODO: Re-enable. Disabled for now because of proxy. Container killed after inactivity automatically. - */ - // Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId); - // try { - // $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - // $executor->deleteRuntime($projectId, $deploymentId); - // } catch (Throwable $th) { - // Console::error($th->getMessage()); - // } + // TODO: Request executor to delete runtime } diff --git a/app/workers/functions.php b/app/workers/functions.php index 8a3940de8a..2d7f4565c4 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -164,14 +164,13 @@ $execute = function ( try { $executionResponse = $executor->createExecution( projectId: $project->getId(), - deploymentId: $deploymentId, - path: $build->getAttribute('outputPath', ''), - vars: $vars, + deploymentId: $deployment->getId(), + payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', + variables: $vars, + timeout: $function->getAttribute('timeout', 0), + image: $runtime['image'], + source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), - data: $vars['APPWRITE_FUNCTION_DATA'] ?? '', - runtime: $function->getAttribute('runtime', ''), - baseImage: $runtime['image'], - timeout: $function->getAttribute('timeout', 0) ); /** Update execution status */ diff --git a/bin/worker-functions b/bin/worker-functions index 143b431b0c..b22ee65d39 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file +php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file diff --git a/composer.lock b/composer.lock index f042279e4f..3912636eea 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b125039c64ae4cbe0d2a1b57322d0ebe", + "content-hash": "3e24f0f02ec826898d50e4119f9eb226", "packages": [ { "name": "adhocore/jwt", @@ -2357,6 +2357,67 @@ }, "time": "2020-10-24T07:04:59+00:00" }, + { + "name": "utopia-php/queue", + "version": "dev-upgrade-libs", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/queue.git", + "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/310aaac74d2287d3d9450a532658247cdfe5e72c", + "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.14.*", + "utopia-php/framework": "0.*.*" + }, + "require-dev": { + "laravel/pint": "^0.2.3", + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.5.5", + "swoole/ide-helper": "4.8.8", + "workerman/workerman": "^4.0" + }, + "suggest": { + "ext-swoole": "Needed to support Swoole.", + "workerman/workerman": "Needed to support Workerman." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Queue\\": "src/Queue" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A powerful task queue.", + "keywords": [ + "Tasks", + "framework", + "php", + "queue", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/queue/issues", + "source": "https://github.com/utopia-php/queue/tree/upgrade-libs" + }, + "time": "2022-11-15T16:35:56+00:00" + }, { "name": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5216,6 +5277,12 @@ } ], "aliases": [ + { + "package": "utopia-php/queue", + "version": "dev-upgrade-libs", + "alias": "0.4.1", + "alias_normalized": "0.4.1.0" + }, { "package": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5225,6 +5292,7 @@ ], "minimum-stability": "stable", "stability-flags": { + "utopia-php/queue": 20, "utopia-php/registry": 20 }, "prefer-stable": false, @@ -5250,5 +5318,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } From 44a82de09bfa23b9eda9a7dc03448d6a8a9b5424 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 19:13:17 +0100 Subject: [PATCH 10/31] Refactor func event triggering --- app/cli.php | 5 +++ app/controllers/api/functions.php | 26 +++++++-------- app/controllers/shared/api.php | 14 +++++--- app/init.php | 15 +++++---- app/worker.php | 8 ++++- app/workers/builds.php | 19 ++++++++--- app/workers/functions.php | 41 ++++++++++-------------- composer.json | 2 +- composer.lock | 21 ++++-------- src/Appwrite/Event/Func.php | 40 +++++++---------------- src/Appwrite/Platform/Tasks/Schedule.php | 24 +++++++------- 11 files changed, 106 insertions(+), 109 deletions(-) diff --git a/app/cli.php b/app/cli.php index 502ee77b75..544148c8d5 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,6 +3,7 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; +use Appwrite\Event\Func; use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; @@ -109,6 +110,10 @@ CLI::setResource('influxdb', function (Registry $register) { return $database; }, ['register']); +CLI::setResource('functions', function (Group $pools) { + return new Func($pools->get('queue')->pop()->getResource()); +}, ['pools']); + CLI::setResource('logError', function (Registry $register) { return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d9f63a93be..02c3fce885 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -5,6 +5,7 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Func; use Appwrite\Event\Validator\Event as ValidatorEvent; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\CustomId; @@ -41,7 +42,6 @@ use Utopia\CLI\Console; use Utopia\Database\Validator\Roles; use Utopia\Validator\Boolean; use Utopia\Database\Exception\Duplicate as DuplicateException; -use Utopia\Queue\Client as QueueClient; include_once __DIR__ . '/../shared/api.php'; @@ -1061,7 +1061,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('usage') ->inject('mode') ->inject('pools') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools) { + ->inject('functions') + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools, Func $functions) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1149,18 +1150,15 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $queueForFunctions = new QueueClient(Event::FUNCTIONS_QUEUE_NAME, $pools->get('queue')->pop()->getResource()); - $queueForFunctions->enqueue([ - 'type' => 'http', - 'value' => [ - 'type' => 'http', - 'execution' => $execution, - 'function' => $function, - 'data' => $data, - 'jwt' => $jwt, - 'project' => $project, - 'user' => $user - ]]); + $functions + ->setType('http') + ->setExecution($execution) + ->setFunction($function) + ->setData($data) + ->setJWT($jwt) + ->setProject($project) + ->setUser($user) + ->trigger(); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e07f405140..58dd2f16ef 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -5,6 +5,7 @@ use Appwrite\Event\Audit; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Usage\Stats; @@ -251,7 +252,8 @@ App::shutdown() ->inject('database') ->inject('mode') ->inject('dbForProject') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject) use ($parseLabel) { + ->inject('functions') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $functions) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -262,9 +264,13 @@ App::shutdown() /** * Trigger functions. */ - $events - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + $functions + ->setData(\json_encode($events->getPayload())) + ->setProject($events->getProject()) + ->setUser($events->getUser()) + ->setEvent($events->getEvent()) + ->setParam('functionId', $events->getParam('functionId')) + ->setParam('executionId', $events->getParam('executionId')) ->trigger(); /** diff --git a/app/init.php b/app/init.php index b861297cef..19cea371df 100644 --- a/app/init.php +++ b/app/init.php @@ -71,6 +71,7 @@ use Utopia\Pools\Group; use Utopia\Pools\Pool; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; +use Appwrite\Event\Func; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; @@ -843,6 +844,9 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); +App::setResource('functions', function (Group $pools) { + return new Func($pools->get('queue')->pop()->getResource()); +}, ['pools']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); @@ -1023,13 +1027,12 @@ App::setResource('console', function () { }, []); App::setResource('queue', function () { - $fallbackForRedis = AppwriteURL::unparse([ - 'scheme' => 'redis', - 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), - 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), - 'user' => App::getEnv('_APP_REDIS_USER', ''), - 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), ]); $connection = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); diff --git a/app/worker.php b/app/worker.php index 4e484e1545..dabb91f1d8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,6 +2,7 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Event\Func; use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -68,7 +69,12 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); -App::setResource('logger', function ($register) { +Server::setResource('functions', function (Registry $register) { + $pools = $register->get('pools'); + return new Func($pools->get('queue')->pop()->getResource()); +}, ['register']); + +Server::setResource('logger', function ($register) { return $register->get('logger'); }, ['register']); diff --git a/app/workers/builds.php b/app/workers/builds.php index babf2874d6..d52329b107 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -1,6 +1,7 @@ setAttribute('status', 'building'); $build = $dbForProject->updateDocument('builds', $buildId, $build); + $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); + /** Trigger Webhook */ $deploymentModel = new Deployment(); @@ -114,14 +117,22 @@ class BuildsV1 extends Worker ->setEvent('functions.[functionId].deployments.[deploymentId].update') ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) + ->setPayload($data) ->trigger(); /** Trigger Functions */ - $deploymentUpdate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + global $register; + $pools = $register->get('pools'); + $connection = $pools->get('queue')->pop(); + $functions = new Func($connection->getResource()); + $functions + ->setData(\json_encode($data)) + ->setProject($project) + ->setEvent('functions.[functionId].deployments.[deploymentId].update') + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) ->trigger(); + $connection->reclaim(); /** Trigger Realtime */ $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ diff --git a/app/workers/functions.php b/app/workers/functions.php index 2d7f4565c4..bad6e7d640 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/../worker.php'; use Utopia\Queue; use Utopia\Queue\Message; use Appwrite\Event\Event; +use Appwrite\Event\Func; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Usage\Stats; use Appwrite\Utopia\Response\Model\Execution; @@ -33,6 +34,7 @@ $execute = function ( Document $project, Document $function, Database $dbForProject, + Func $functions, string $trigger, string $executionId = null, string $event = null, @@ -206,9 +208,13 @@ $execute = function ( ->trigger(); /** Trigger Functions */ - $executionUpdate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + $functions + ->setData($data) + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) ->trigger(); /** Trigger realtime event */ @@ -259,9 +265,10 @@ $server = new Queue\Server($adapter); $server->job() ->inject('message') ->inject('dbForProject') - ->action(function (Message $message, Database $dbForProject) use ($execute) { - $args = $message->getPayload()['value'] ?? []; - $type = $message->getPayload()['type'] ?? ''; + ->inject('functions') + ->action(function (Message $message, Database $dbForProject, Func $functions) use ($execute) { + $args = $message->getPayload() ?? []; + $type = $args['type'] ?? ''; $events = $args['events'] ?? []; $project = new Document($args['project'] ?? []); $user = new Document($args['user'] ?? []); @@ -317,34 +324,20 @@ $server->job() /** * Handle Schedule and HTTP execution. */ - $user = new Document($args['user'] ?? []); $project = new Document($args['project'] ?? []); - $execution = new Document($args['execution'] ?? []); $function = new Document($args['function'] ?? []); switch ($type) { case 'http': $jwt = $args['jwt'] ?? ''; $data = $args['data'] ?? ''; + $execution = new Document($args['execution'] ?? []); + $user = new Document($args['user'] ?? []); $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); - call_user_func($execute, $project, $function, $dbForProject, 'http', $execution->getId(), null, null, $data, $user, $jwt); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt); break; - case 'schedule': - /* - * 1. Get Original Task - * 2. Check for updates - * If has updates skip task and don't reschedule - * If status not equal to play skip task - * 3. Check next run date, update task and add new job at the given date - * 4. Execute task (set optional timeout) - * 5. Update task response to log - * On success reset error count - * On failure add error count - * If error count bigger than allowed change status to pause - */ - - call_user_func($execute, $project, $function, $dbForProject, 'schedule', null, null, null, null, null, null); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null); break; } }); diff --git a/composer.json b/composer.json index d4c06d7614..08ede65dcf 100644 --- a/composer.json +++ b/composer.json @@ -53,7 +53,7 @@ "utopia-php/domains": "1.1.*", "utopia-php/framework": "0.25.*", "utopia-php/image": "0.5.*", - "utopia-php/queue": "dev-upgrade-libs as 0.4.1", + "utopia-php/queue": "0.4.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", "utopia-php/orchestration": "0.9.*", diff --git a/composer.lock b/composer.lock index 3912636eea..674e42a5ca 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3e24f0f02ec826898d50e4119f9eb226", + "content-hash": "f2faa670abaa356f9b14e4f1c3542b33", "packages": [ { "name": "adhocore/jwt", @@ -2359,16 +2359,16 @@ }, { "name": "utopia-php/queue", - "version": "dev-upgrade-libs", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c" + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/310aaac74d2287d3d9450a532658247cdfe5e72c", - "reference": "310aaac74d2287d3d9450a532658247cdfe5e72c", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", "shasum": "" }, "require": { @@ -2414,9 +2414,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/upgrade-libs" + "source": "https://github.com/utopia-php/queue/tree/0.4.1" }, - "time": "2022-11-15T16:35:56+00:00" + "time": "2022-11-15T16:56:37+00:00" }, { "name": "utopia-php/registry", @@ -5277,12 +5277,6 @@ } ], "aliases": [ - { - "package": "utopia-php/queue", - "version": "dev-upgrade-libs", - "alias": "0.4.1", - "alias_normalized": "0.4.1.0" - }, { "package": "utopia-php/registry", "version": "dev-feat-allow-params", @@ -5292,7 +5286,6 @@ ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/queue": 20, "utopia-php/registry": 20 }, "prefer-stable": false, diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index b7531cf475..514eb5df74 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -6,6 +6,8 @@ use DateTime; use Resque; use ResqueScheduler; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Func extends Event { @@ -15,7 +17,7 @@ class Func extends Event protected ?Document $function = null; protected ?Document $execution = null; - public function __construct() + public function __construct(protected Connection $connection) { parent::__construct(Event::FUNCTIONS_QUEUE_NAME, Event::FUNCTIONS_CLASS_NAME); } @@ -143,36 +145,16 @@ class Func extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ - 'project' => $this->project, - 'user' => $this->user, - 'function' => $this->function, - 'execution' => $this->execution, - 'type' => $this->type, - 'jwt' => $this->jwt, - 'payload' => $this->payload, - 'data' => $this->data - ]); - } + $queue = new Client(Event::FUNCTIONS_QUEUE_NAME, $this->connection); - /** - * Schedules the function event and schedules it in the functions worker queue. - * - * @param \DateTime|int $at - * @return void - * @throws \Resque_Exception - * @throws \ResqueScheduler_InvalidTimestampException - */ - public function schedule(DateTime|int $at): void - { - ResqueScheduler::enqueueAt($at, $this->queue, $this->class, [ - 'project' => $this->project, - 'user' => $this->user, - 'function' => $this->function, - 'execution' => $this->execution, + return $queue->enqueue([ 'type' => $this->type, - 'payload' => $this->payload, - 'data' => $this->data + 'execution' => $this->execution, + 'function' => $this->function, + 'data' => $this->data, + 'jwt' => $this->jwt, + 'project' => $this->project, + 'user' => $this->user ]); } } diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index 39e0686da5..88fd0a9a80 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -14,6 +14,7 @@ use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Queue\Client as Worker; use Appwrite\Event\Event; +use Appwrite\Event\Func; use function Swoole\Coroutine\run; @@ -203,6 +204,9 @@ class Schedule extends Action \go(function () use ($delay, $schedules, $scheduleKeys, $pools) { \sleep($delay); // in seconds + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + foreach ($scheduleKeys as $scheduleKey) { // Ensure schedule was not deleted if (!isset($schedules[$scheduleKey])) { @@ -211,20 +215,16 @@ class Schedule extends Action $schedule = $schedules[$scheduleKey]; - $queue = $pools->get('queue')->pop(); + $functions = new Func($connection); - $worker = new Worker(Event::FUNCTIONS_QUEUE_NAME, $queue->getResource()); - $worker - ->enqueue([ - 'type' => 'schedule', - 'value' => [ - 'project' => $schedule['project'], - 'function' => $schedule['function'], - ] - ]); - - $queue->reclaim(); + $functions + ->setType('schedule') + ->setFunction($schedule['function']) + ->setProject($schedule['project']) + ->trigger(); } + + $queue->reclaim(); }); } From 6bf370a0581047b786564d9293e5f6267be7107c Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 15 Nov 2022 19:43:40 +0100 Subject: [PATCH 11/31] QA bug fixes --- app/controllers/api/functions.php | 3 +-- app/worker.php | 2 +- app/workers/builds.php | 4 ++-- app/workers/functions.php | 6 +++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 02c3fce885..7204271d0f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1157,8 +1157,7 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/worker.php b/app/worker.php index dabb91f1d8..564a577c71 100644 --- a/app/worker.php +++ b/app/worker.php @@ -33,7 +33,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) }, ['cache', 'register']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) { - $args = $message->getPayload()['value'] ?? []; + $args = $message->getPayload() ?? []; $project = new Document($args['project'] ?? []); if ($project->isEmpty() || $project->getId() === 'console') { diff --git a/app/workers/builds.php b/app/workers/builds.php index d52329b107..315b13a179 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -106,11 +106,11 @@ class BuildsV1 extends Worker $build->setAttribute('status', 'building'); $build = $dbForProject->updateDocument('builds', $buildId, $build); - $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); - /** Trigger Webhook */ $deploymentModel = new Deployment(); + $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); + $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); $deploymentUpdate ->setProject($project) diff --git a/app/workers/functions.php b/app/workers/functions.php index bad6e7d640..e0c9e64961 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -43,11 +43,11 @@ $execute = function ( ?Document $user = null, string $jwt = null ) use ($executor, $register) { - $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); + /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -209,7 +209,7 @@ $execute = function ( /** Trigger Functions */ $functions - ->setData($data) + ->setData($data ?? '') ->setProject($project) ->setUser($user) ->setEvent('functions.[functionId].executions.[executionId].update') @@ -333,7 +333,7 @@ $server->job() $data = $args['data'] ?? ''; $execution = new Document($args['execution'] ?? []); $user = new Document($args['user'] ?? []); - $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); + // $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt); break; case 'schedule': From 4a92db4dc393fa3e382c308f7f4ff5857d5565aa Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 09:47:46 +0530 Subject: [PATCH 12/31] feat: check async execution --- app/controllers/api/functions.php | 8 +- app/worker.php | 33 ++-- app/workers/functions.php | 99 ++++++------ composer.lock | 2 +- temp/appwrite.json | 21 +++ temp/functions/My Awesome Function/.gitignore | 149 ++++++++++++++++++ temp/functions/My Awesome Function/README.md | 47 ++++++ .../My Awesome Function/package.json | 15 ++ .../My Awesome Function/src/index.js | 46 ++++++ 9 files changed, 352 insertions(+), 68 deletions(-) create mode 100644 temp/appwrite.json create mode 100644 temp/functions/My Awesome Function/.gitignore create mode 100644 temp/functions/My Awesome Function/README.md create mode 100644 temp/functions/My Awesome Function/package.json create mode 100644 temp/functions/My Awesome Function/src/index.js diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7204271d0f..04da1234a1 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1217,10 +1217,10 @@ App::post('/v1/functions/:functionId/executions') // TODO revise this later using route label $usage - ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')); // ms + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')); // ms $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); diff --git a/app/worker.php b/app/worker.php index 564a577c71..110a9d81da 100644 --- a/app/worker.php +++ b/app/worker.php @@ -20,37 +20,37 @@ Server::setResource('register', fn() => $register); Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { $pools = $register->get('pools'); - $dbAdapter = $pools + $database = $pools ->get('console') ->pop() ->getResource() ; - $database = new Database($dbAdapter, $cache); - $database->setNamespace('console'); + $adapter = new Database($database, $cache); + $adapter->setNamespace('console'); - return $database; + return $adapter; }, ['cache', 'register']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) { - $args = $message->getPayload() ?? []; - $project = new Document($args['project'] ?? []); + $payload = $message->getPayload() ?? []; + $project = new Document($payload['project'] ?? []); if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } $pools = $register->get('pools'); - $dbAdapter = $pools + $database = $pools ->get($project->getAttribute('database')) ->pop() ->getResource() ; - $database = new Database($dbAdapter, $cache); - $database->setNamespace('_' . $project->getInternalId()); + $adapter = new Database($database, $cache); + $adapter->setNamespace('_' . $project->getInternalId()); - return $database; + return $adapter; }, ['cache', 'register', 'message', 'dbForConsole']); Server::setResource('cache', function (Registry $register) { @@ -71,19 +71,26 @@ Server::setResource('cache', function (Registry $register) { Server::setResource('functions', function (Registry $register) { $pools = $register->get('pools'); - return new Func($pools->get('queue')->pop()->getResource()); + return new Func( + $pools + ->get('queue') + ->pop() + ->getResource() + ); }, ['register']); Server::setResource('logger', function ($register) { return $register->get('logger'); }, ['register']); +Server::setResource('statsd', function ($register) { + return $register->get('statsd'); +}, ['register']); $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); - $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); $workerNumber = 1; -Runtime::enableCoroutine(SWOOLE_HOOK_ALL); +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); \ No newline at end of file diff --git a/app/workers/functions.php b/app/workers/functions.php index e0c9e64961..02f3932c12 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,13 +2,13 @@ require_once __DIR__ . '/../worker.php'; -use Utopia\Queue; use Utopia\Queue\Message; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Usage\Stats; use Appwrite\Utopia\Response\Model\Execution; +use Domnikl\Statsd\Client; use Executor\Executor; use Utopia\App; use Utopia\CLI\Console; @@ -21,14 +21,16 @@ use Utopia\Database\Query; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; +use Utopia\Queue\Adapter\Swoole; +use Utopia\Queue\Server; Authorization::disable(); Authorization::setDefaultStatus(false); global $connection; global $workerNumber; - -$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); +$adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); +$server = new Server($adapter); $execute = function ( Document $project, @@ -41,12 +43,14 @@ $execute = function ( string $eventData = null, string $data = null, ?Document $user = null, - string $jwt = null -) use ($executor, $register) { + string $jwt = null, + Client $statsd +) { + $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); - + var_dump("Deployment ID : ", $deploymentId); /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -163,6 +167,7 @@ $execute = function ( ]); /** Execute function */ + $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { $executionResponse = $executor->createExecution( projectId: $project->getId(), @@ -223,7 +228,7 @@ $execute = function ( 'executionId' => $execution->getId() ]); $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $execution ); @@ -243,12 +248,11 @@ $execute = function ( ); /** Update usage stats */ - global $register; if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $statsd = $register->get('statsd'); $usage = new Stats($statsd); $usage ->setParam('projectId', $project->getId()) + ->setParam('projectInternalId', $project->getInternalId()) ->setParam('functionId', $function->getId()) ->setParam('executions.{scope}.compute', 1) ->setParam('executionStatus', $execution->getAttribute('status', '')) @@ -259,21 +263,25 @@ $execute = function ( } }; -$adapter = new Queue\Adapter\Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); -$server = new Queue\Server($adapter); - $server->job() ->inject('message') ->inject('dbForProject') ->inject('functions') - ->action(function (Message $message, Database $dbForProject, Func $functions) use ($execute) { - $args = $message->getPayload() ?? []; - $type = $args['type'] ?? ''; - $events = $args['events'] ?? []; - $project = new Document($args['project'] ?? []); - $user = new Document($args['user'] ?? []); - // Where $payload comes from - $payload = json_encode($args['payload'] ?? []); + ->inject('statsd') + ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd) use ($execute) { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + var_dump(json_encode($payload)); + + $type = $payload['type'] ?? ''; + $events = $payload['events'] ?? []; + $project = new Document($payload['project'] ?? []); + $data = $payload['data'] ?? ''; + $user = new Document($payload['user'] ?? []); if ($project->getId() === 'console') { return; @@ -284,38 +292,31 @@ $server->job() */ if (!empty($events)) { $limit = 30; - $sum = $limit; - $total = 0; - $latestDocument = null; + $sum = 30; + $offset = 0; + $functions = []; + /** @var Document[] $functions */ - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - $results = $dbForProject->find('functions', \array_merge($paginationQueries, [ - Query::orderAsc('name') - ])); + while ($sum >= $limit) { + $functions = $dbForProject->find('functions', [ + Query::limit($limit), + Query::offset($offset), + Query::orderAsc('name'), + ]); - $sum = count($results); - $total = $total + $sum; + $sum = \count($functions); + $offset = $offset + $limit; Console::log('Fetched ' . $sum . ' functions...'); - foreach ($results as $function) { + foreach ($functions as $function) { if (!array_intersect($events, $function->getAttribute('events', []))) { continue; } - Console::success('Iterating function: ' . $function->getAttribute('name')); - - // As event, pass first, most verbose event pattern - call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null); - + call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null, $statsd); Console::success('Triggered function: ' . $events[0]); } - - $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; } return; @@ -324,20 +325,18 @@ $server->job() /** * Handle Schedule and HTTP execution. */ - $project = new Document($args['project'] ?? []); - $function = new Document($args['function'] ?? []); + $function = new Document($payload['function'] ?? []); + var_dump($function); switch ($type) { case 'http': - $jwt = $args['jwt'] ?? ''; - $data = $args['data'] ?? ''; - $execution = new Document($args['execution'] ?? []); - $user = new Document($args['user'] ?? []); - // $function = $dbForProject->getDocument('functions', $execution->getAttribute('functionId')); - call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt); + $jwt = $payload['jwt'] ?? ''; + $execution = new Document($payload['execution'] ?? []); + $user = new Document($payload['user'] ?? []); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt, $statsd); break; case 'schedule': - call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null); + call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null, $statsd); break; } }); diff --git a/composer.lock b/composer.lock index 674e42a5ca..d41a4a1cf9 100644 --- a/composer.lock +++ b/composer.lock @@ -5311,5 +5311,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.3.0" } diff --git a/temp/appwrite.json b/temp/appwrite.json new file mode 100644 index 0000000000..643a821415 --- /dev/null +++ b/temp/appwrite.json @@ -0,0 +1,21 @@ +{ + "projectId": "6374484424b42c83155c", + "projectName": "Default", + "functions": [ + { + "$id": "637449ac7066fa0d8cde", + "name": "My Awesome Function", + "runtime": "node-14.5", + "path": "functions/My Awesome Function", + "entrypoint": "src/index.js", + "ignore": [ + "node_modules", + ".npm" + ], + "execute": [], + "events": [], + "schedule": "", + "timeout": 15 + } + ] +} \ No newline at end of file diff --git a/temp/functions/My Awesome Function/.gitignore b/temp/functions/My Awesome Function/.gitignore new file mode 100644 index 0000000000..2551987d6d --- /dev/null +++ b/temp/functions/My Awesome Function/.gitignore @@ -0,0 +1,149 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/node +# Edit at https://www.toptal.com/developers/gitignore?templates=node + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +### Node Patch ### +# Serverless Webpack directories +.webpack/ + +# Optional stylelint cache + +# SvelteKit build / generate output +.svelte-kit + +# End of https://www.toptal.com/developers/gitignore/api/node + +# OS +## Mac +.DS_Store diff --git a/temp/functions/My Awesome Function/README.md b/temp/functions/My Awesome Function/README.md new file mode 100644 index 0000000000..0eb0d2ad04 --- /dev/null +++ b/temp/functions/My Awesome Function/README.md @@ -0,0 +1,47 @@ +# My Awesome Function + +Welcome to the documentation of this function 👋 We strongly recommend keeping this file in sync with your function's logic to make sure anyone can easily understand your function in the future. If you don't need documentation, you can remove this file. + +## 🤖 Documentation + +Simple function similar to typical "hello world" example, but instead, we return a simple JSON that tells everyone how awesome developers are. + + + +_Example input:_ + +This function expects no input + + + +_Example output:_ + + + +```json +{ + "areDevelopersAwesome": true +} +``` + +## 📝 Environment Variables + +List of environment variables used by this cloud function: + +- **APPWRITE_FUNCTION_ENDPOINT** - Endpoint of Appwrite project +- **APPWRITE_FUNCTION_API_KEY** - Appwrite API Key + + +## 🚀 Deployment + +There are two ways of deploying the Appwrite function, both having the same results, but each using a different process. We highly recommend using CLI deployment to achieve the best experience. + +### Using CLI + +Make sure you have [Appwrite CLI](https://appwrite.io/docs/command-line#installation) installed, and you have successfully logged into your Appwrite server. To make sure Appwrite CLI is ready, you can use the command `appwrite client --debug` and it should respond with green text `✓ Success`. + +Make sure you are in the same folder as your `appwrite.json` file and run `appwrite deploy function` to deploy your function. You will be prompted to select which functions you want to deploy. + +### Manual using tar.gz + +Manual deployment has no requirements and uses Appwrite Console to deploy the tag. First, enter the folder of your function. Then, create a tarball of the whole folder and gzip it. After creating `.tar.gz` file, visit Appwrite Console, click on the `Deploy Tag` button and switch to the `Manual` tab. There, set the `entrypoint` to `src/index.js`, and upload the file we just generated. diff --git a/temp/functions/My Awesome Function/package.json b/temp/functions/My Awesome Function/package.json new file mode 100644 index 0000000000..fa252199b9 --- /dev/null +++ b/temp/functions/My Awesome Function/package.json @@ -0,0 +1,15 @@ +{ + "name": "appwrite-function", + "version": "1.0.0", + "description": "", + "main": "src/index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "node-appwrite": "^8.0.0" + } +} diff --git a/temp/functions/My Awesome Function/src/index.js b/temp/functions/My Awesome Function/src/index.js new file mode 100644 index 0000000000..4ed0028cac --- /dev/null +++ b/temp/functions/My Awesome Function/src/index.js @@ -0,0 +1,46 @@ +const sdk = require("node-appwrite"); + +/* + 'req' variable has: + 'headers' - object with request headers + 'payload' - request body data as a string + 'variables' - object with function variables + + 'res' variable has: + 'send(text, status)' - function to return text response. Status code defaults to 200 + 'json(obj, status)' - function to return JSON response. Status code defaults to 200 + + If an error is thrown, a response with code 500 will be returned. +*/ + +module.exports = async function (req, res) { + const client = new sdk.Client(); + + // You can remove services you don't use + const account = new sdk.Account(client); + const avatars = new sdk.Avatars(client); + const database = new sdk.Databases(client); + const functions = new sdk.Functions(client); + const health = new sdk.Health(client); + const locale = new sdk.Locale(client); + const storage = new sdk.Storage(client); + const teams = new sdk.Teams(client); + const users = new sdk.Users(client); + + if ( + !req.variables['APPWRITE_FUNCTION_ENDPOINT'] || + !req.variables['APPWRITE_FUNCTION_API_KEY'] + ) { + console.warn("Environment variables are not set. Function cannot use Appwrite SDK."); + } else { + client + .setEndpoint(req.variables['APPWRITE_FUNCTION_ENDPOINT']) + .setProject(req.variables['APPWRITE_FUNCTION_PROJECT_ID']) + .setKey(req.variables['APPWRITE_FUNCTION_API_KEY']) + .setSelfSigned(true); + } + + res.json({ + areDevelopersAwesome: true, + }); +}; From 3bd3b8d8e9379840a98ab782e32368578d2ba474 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 09:48:11 +0530 Subject: [PATCH 13/31] feat: check async execution --- temp/appwrite.json | 21 --- temp/functions/My Awesome Function/.gitignore | 149 ------------------ temp/functions/My Awesome Function/README.md | 47 ------ .../My Awesome Function/package.json | 15 -- .../My Awesome Function/src/index.js | 46 ------ 5 files changed, 278 deletions(-) delete mode 100644 temp/appwrite.json delete mode 100644 temp/functions/My Awesome Function/.gitignore delete mode 100644 temp/functions/My Awesome Function/README.md delete mode 100644 temp/functions/My Awesome Function/package.json delete mode 100644 temp/functions/My Awesome Function/src/index.js diff --git a/temp/appwrite.json b/temp/appwrite.json deleted file mode 100644 index 643a821415..0000000000 --- a/temp/appwrite.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "projectId": "6374484424b42c83155c", - "projectName": "Default", - "functions": [ - { - "$id": "637449ac7066fa0d8cde", - "name": "My Awesome Function", - "runtime": "node-14.5", - "path": "functions/My Awesome Function", - "entrypoint": "src/index.js", - "ignore": [ - "node_modules", - ".npm" - ], - "execute": [], - "events": [], - "schedule": "", - "timeout": 15 - } - ] -} \ No newline at end of file diff --git a/temp/functions/My Awesome Function/.gitignore b/temp/functions/My Awesome Function/.gitignore deleted file mode 100644 index 2551987d6d..0000000000 --- a/temp/functions/My Awesome Function/.gitignore +++ /dev/null @@ -1,149 +0,0 @@ - -# Created by https://www.toptal.com/developers/gitignore/api/node -# Edit at https://www.toptal.com/developers/gitignore?templates=node - -### Node ### -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local -.env.local - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp and cache directory -.temp - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# yarn v2 -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.* - -### Node Patch ### -# Serverless Webpack directories -.webpack/ - -# Optional stylelint cache - -# SvelteKit build / generate output -.svelte-kit - -# End of https://www.toptal.com/developers/gitignore/api/node - -# OS -## Mac -.DS_Store diff --git a/temp/functions/My Awesome Function/README.md b/temp/functions/My Awesome Function/README.md deleted file mode 100644 index 0eb0d2ad04..0000000000 --- a/temp/functions/My Awesome Function/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# My Awesome Function - -Welcome to the documentation of this function 👋 We strongly recommend keeping this file in sync with your function's logic to make sure anyone can easily understand your function in the future. If you don't need documentation, you can remove this file. - -## 🤖 Documentation - -Simple function similar to typical "hello world" example, but instead, we return a simple JSON that tells everyone how awesome developers are. - - - -_Example input:_ - -This function expects no input - - - -_Example output:_ - - - -```json -{ - "areDevelopersAwesome": true -} -``` - -## 📝 Environment Variables - -List of environment variables used by this cloud function: - -- **APPWRITE_FUNCTION_ENDPOINT** - Endpoint of Appwrite project -- **APPWRITE_FUNCTION_API_KEY** - Appwrite API Key - - -## 🚀 Deployment - -There are two ways of deploying the Appwrite function, both having the same results, but each using a different process. We highly recommend using CLI deployment to achieve the best experience. - -### Using CLI - -Make sure you have [Appwrite CLI](https://appwrite.io/docs/command-line#installation) installed, and you have successfully logged into your Appwrite server. To make sure Appwrite CLI is ready, you can use the command `appwrite client --debug` and it should respond with green text `✓ Success`. - -Make sure you are in the same folder as your `appwrite.json` file and run `appwrite deploy function` to deploy your function. You will be prompted to select which functions you want to deploy. - -### Manual using tar.gz - -Manual deployment has no requirements and uses Appwrite Console to deploy the tag. First, enter the folder of your function. Then, create a tarball of the whole folder and gzip it. After creating `.tar.gz` file, visit Appwrite Console, click on the `Deploy Tag` button and switch to the `Manual` tab. There, set the `entrypoint` to `src/index.js`, and upload the file we just generated. diff --git a/temp/functions/My Awesome Function/package.json b/temp/functions/My Awesome Function/package.json deleted file mode 100644 index fa252199b9..0000000000 --- a/temp/functions/My Awesome Function/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "appwrite-function", - "version": "1.0.0", - "description": "", - "main": "src/index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "node-appwrite": "^8.0.0" - } -} diff --git a/temp/functions/My Awesome Function/src/index.js b/temp/functions/My Awesome Function/src/index.js deleted file mode 100644 index 4ed0028cac..0000000000 --- a/temp/functions/My Awesome Function/src/index.js +++ /dev/null @@ -1,46 +0,0 @@ -const sdk = require("node-appwrite"); - -/* - 'req' variable has: - 'headers' - object with request headers - 'payload' - request body data as a string - 'variables' - object with function variables - - 'res' variable has: - 'send(text, status)' - function to return text response. Status code defaults to 200 - 'json(obj, status)' - function to return JSON response. Status code defaults to 200 - - If an error is thrown, a response with code 500 will be returned. -*/ - -module.exports = async function (req, res) { - const client = new sdk.Client(); - - // You can remove services you don't use - const account = new sdk.Account(client); - const avatars = new sdk.Avatars(client); - const database = new sdk.Databases(client); - const functions = new sdk.Functions(client); - const health = new sdk.Health(client); - const locale = new sdk.Locale(client); - const storage = new sdk.Storage(client); - const teams = new sdk.Teams(client); - const users = new sdk.Users(client); - - if ( - !req.variables['APPWRITE_FUNCTION_ENDPOINT'] || - !req.variables['APPWRITE_FUNCTION_API_KEY'] - ) { - console.warn("Environment variables are not set. Function cannot use Appwrite SDK."); - } else { - client - .setEndpoint(req.variables['APPWRITE_FUNCTION_ENDPOINT']) - .setProject(req.variables['APPWRITE_FUNCTION_PROJECT_ID']) - .setKey(req.variables['APPWRITE_FUNCTION_API_KEY']) - .setSelfSigned(true); - } - - res.json({ - areDevelopersAwesome: true, - }); -}; From 586fd46bcdb84df6d4450eb3d6eacaaa34427e39 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 09:49:35 +0530 Subject: [PATCH 14/31] feat: check async execution --- app/workers/functions.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 02f3932c12..67fb716e54 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -279,9 +279,11 @@ $server->job() $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; - $project = new Document($payload['project'] ?? []); $data = $payload['data'] ?? ''; + $project = new Document($payload['project'] ?? []); + $function = new Document($payload['function'] ?? []); $user = new Document($payload['user'] ?? []); + var_dump("Function : ", $function); if ($project->getId() === 'console') { return; @@ -325,9 +327,6 @@ $server->job() /** * Handle Schedule and HTTP execution. */ - $function = new Document($payload['function'] ?? []); - var_dump($function); - switch ($type) { case 'http': $jwt = $payload['jwt'] ?? ''; From 5abe9ad73cfb46a5120ea779aa73e312643202de Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 10:10:34 +0530 Subject: [PATCH 15/31] feat: refactor execute function to a resource --- app/workers/functions.php | 503 ++++++++++++++++++++------------------ 1 file changed, 271 insertions(+), 232 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 67fb716e54..a34f0aec99 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -32,243 +32,246 @@ global $workerNumber; $adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); $server = new Server($adapter); -$execute = function ( - Document $project, - Document $function, - Database $dbForProject, - Func $functions, - string $trigger, - string $executionId = null, - string $event = null, - string $eventData = null, - string $data = null, - ?Document $user = null, - string $jwt = null, - Client $statsd -) { +Server::setResource('execute', function () { + return function ( + Document $project, + Document $function, + Database $dbForProject, + Func $functions, + string $trigger, + string $executionId = null, + string $event = null, + string $eventData = null, + string $data = null, + ?Document $user = null, + string $jwt = null, + Client $statsd + ) { - $user ??= new Document(); - $functionId = $function->getId(); - $deploymentId = $function->getAttribute('deployment', ''); - var_dump("Deployment ID : ", $deploymentId); - - /** Check if deployment exists */ - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - if ($deployment->getAttribute('resourceId') !== $functionId) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function'); - } - - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found. Create deployment before trying to execute a function'); - } - - /** Check if build has exists */ - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - if ($build->isEmpty()) { - throw new Exception('Build not found'); - } - - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready'); - } - - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); - - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); + $user ??= new Document(); + $functionId = $function->getId(); + $deploymentId = $function->getAttribute('deployment', ''); + var_dump("Deployment ID : ", $deploymentId); + + /** Check if deployment exists */ + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if ($deployment->getAttribute('resourceId') !== $functionId) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready'); - } - - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); - - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); + + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { - $carry[$var->getAttribute('key')] = $var->getAttribute('value'); - return $carry; - }, []); - - /** Collect environment variables */ - $vars = \array_merge($vars, [ - 'APPWRITE_FUNCTION_ID' => $functionId, - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, - 'APPWRITE_FUNCTION_TRIGGER' => $trigger, - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_EVENT' => $event ?? '', - 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', - 'APPWRITE_FUNCTION_DATA' => $data ?? '', - 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', - 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', - ]); - - /** Execute function */ - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - try { - $executionResponse = $executor->createExecution( - projectId: $project->getId(), - deploymentId: $deployment->getId(), - payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', - variables: $vars, - timeout: $function->getAttribute('timeout', 0), - image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), - entrypoint: $deployment->getAttribute('entrypoint', ''), - ); - - /** Update execution status */ - $execution - ->setAttribute('status', $executionResponse['status']) - ->setAttribute('statusCode', $executionResponse['statusCode']) - ->setAttribute('response', $executionResponse['response']) - ->setAttribute('stdout', $executionResponse['stdout']) - ->setAttribute('stderr', $executionResponse['stderr']) - ->setAttribute('duration', $executionResponse['duration']); - } catch (\Throwable $th) { - $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); - $execution - ->setAttribute('duration', (float)$interval->format('%s.%f')) - ->setAttribute('status', 'failed') - ->setAttribute('statusCode', $th->getCode()) - ->setAttribute('stderr', $th->getMessage()); - Console::error($th->getMessage()); - } - - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - /** Trigger Webhook */ - $executionModel = new Execution(); - $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); - $executionUpdate - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) - ->trigger(); - - /** Trigger Functions */ - $functions - ->setData($data ?? '') - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->trigger(); - - /** Trigger realtime event */ - $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ - 'functionId' => $function->getId(), - 'executionId' => $execution->getId() - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $execution - ); - Realtime::send( - projectId: 'console', - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - /** Update usage stats */ - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $usage = new Stats($statsd); - $usage - ->setParam('projectId', $project->getId()) - ->setParam('projectInternalId', $project->getInternalId()) + + /** Check if build has exists */ + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + if ($build->isEmpty()) { + throw new Exception('Build not found'); + } + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready'); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception('Build not ready'); + } + + /** Check if runtime is supported */ + $runtimes = Config::getParam('runtimes', []); + + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + + $runtime = $runtimes[$function->getAttribute('runtime')]; + + /** Create execution or update execution status */ + $execution = $dbForProject->getDocument('executions', $executionId ?? ''); + if ($execution->isEmpty()) { + $executionId = ID::unique(); + $execution = $dbForProject->createDocument('executions', new Document([ + '$id' => $executionId, + '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], + 'functionId' => $functionId, + 'deploymentId' => $deploymentId, + 'trigger' => $trigger, + 'status' => 'waiting', + 'statusCode' => 0, + 'response' => '', + 'stderr' => '', + 'duration' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), + ])); + + if ($execution->isEmpty()) { + throw new Exception('Failed to create or read execution'); + } + } + $execution->setAttribute('status', 'processing'); + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $carry[$var->getAttribute('key')] = $var->getAttribute('value'); + return $carry; + }, []); + + /** Collect environment variables */ + $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_ID' => $functionId, + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, + 'APPWRITE_FUNCTION_TRIGGER' => $trigger, + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_FUNCTION_EVENT' => $event ?? '', + 'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '', + 'APPWRITE_FUNCTION_DATA' => $data ?? '', + 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', + 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', + ]); + + /** Execute function */ + $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + try { + $executionResponse = $executor->createExecution( + projectId: $project->getId(), + deploymentId: $deployment->getId(), + payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', + variables: $vars, + timeout: $function->getAttribute('timeout', 0), + image: $runtime['image'], + source: $build->getAttribute('outputPath', ''), + entrypoint: $deployment->getAttribute('entrypoint', ''), + ); + + /** Update execution status */ + $execution + ->setAttribute('status', $executionResponse['status']) + ->setAttribute('statusCode', $executionResponse['statusCode']) + ->setAttribute('response', $executionResponse['response']) + ->setAttribute('stdout', $executionResponse['stdout']) + ->setAttribute('stderr', $executionResponse['stderr']) + ->setAttribute('duration', $executionResponse['duration']); + } catch (\Throwable $th) { + $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt())); + $execution + ->setAttribute('duration', (float)$interval->format('%s.%f')) + ->setAttribute('status', 'failed') + ->setAttribute('statusCode', $th->getCode()) + ->setAttribute('stderr', $th->getMessage()); + Console::error($th->getMessage()); + } + + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + + /** Trigger Webhook */ + $executionModel = new Execution(); + $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); + $executionUpdate + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')) - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ->submit(); - } -}; + ->setParam('executionId', $execution->getId()) + ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) + ->trigger(); + + /** Trigger Functions */ + $functions + ->setData($data ?? '') + ->setProject($project) + ->setUser($user) + ->setEvent('functions.[functionId].executions.[executionId].update') + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) + ->trigger(); + + /** Trigger realtime event */ + $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ + 'functionId' => $function->getId(), + 'executionId' => $execution->getId() + ]); + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $execution + ); + Realtime::send( + projectId: 'console', + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + Realtime::send( + projectId: $project->getId(), + payload: $execution->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + /** Update usage stats */ + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $usage = new Stats($statsd); + $usage + ->setParam('projectId', $project->getId()) + ->setParam('projectInternalId', $project->getInternalId()) + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')) + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit(); + } + }; +}); $server->job() ->inject('message') ->inject('dbForProject') ->inject('functions') ->inject('statsd') - ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd) use ($execute) { + ->inject('execute') + ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd, callable $execute) { $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -276,7 +279,6 @@ $server->job() } var_dump(json_encode($payload)); - $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; $data = $payload['data'] ?? ''; @@ -298,7 +300,6 @@ $server->job() $offset = 0; $functions = []; /** @var Document[] $functions */ - while ($sum >= $limit) { $functions = $dbForProject->find('functions', [ Query::limit($limit), @@ -316,7 +317,19 @@ $server->job() continue; } Console::success('Iterating function: ' . $function->getAttribute('name')); - call_user_func($execute, $project, $function, $dbForProject, 'event', null, $events[0], $payload, null, $user, null, $statsd); + $execute( + statsd: $statsd, + dbForProject: $dbForProject, + project: $project, + function: $function, + trigger: 'event', + event: $events[0], + eventData: $payload, + user: $user, + data: null, + executionId: null, + jwt: null + ); Console::success('Triggered function: ' . $events[0]); } } @@ -332,10 +345,36 @@ $server->job() $jwt = $payload['jwt'] ?? ''; $execution = new Document($payload['execution'] ?? []); $user = new Document($payload['user'] ?? []); - call_user_func($execute, $project, $function, $dbForProject, $functions, 'http', $execution->getId(), null, null, $data, $user, $jwt, $statsd); + $execute( + project: $project, + function: $function, + dbForProject: $dbForProject, + functions: $functions, + trigger: 'http', + executionId: $execution->getId(), + event: null, + eventData: null, + data: $data, + user: $user, + jwt: $jwt, + statsd: $statsd, + ); break; case 'schedule': - call_user_func($execute, $project, $function, $dbForProject, $functions, 'schedule', null, null, null, null, null, null, $statsd); + $execute( + project: $project, + function: $function, + dbForProject: $dbForProject, + functions: $functions, + trigger: 'http', + executionId: null, + event: null, + eventData: null, + data: null, + user: null, + jwt: null, + statsd: $statsd, + ); break; } }); From 8b0a78bb36672ec986df87031e4face4a34ce3ba Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 11:00:57 +0530 Subject: [PATCH 16/31] fix: function events and linter --- app/controllers/api/functions.php | 3 +- app/controllers/shared/api.php | 15 ++-- app/worker.php | 4 +- app/workers/functions.php | 79 ++++++++++--------- src/Appwrite/Event/Func.php | 36 +++++++-- .../Utopia/Response/Model/Execution.php | 1 - 6 files changed, 78 insertions(+), 60 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 04da1234a1..b695d19429 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1157,7 +1157,8 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user); + ->setUser($user) + ->trigger(); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 58dd2f16ef..a12016606f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -129,9 +129,9 @@ App::init() } } - /* - * Background Jobs - */ + /* + * Background Jobs + */ $events ->setEvent($route->getLabel('event', '')) ->setProject($project) @@ -265,12 +265,9 @@ App::shutdown() * Trigger functions. */ $functions - ->setData(\json_encode($events->getPayload())) - ->setProject($events->getProject()) - ->setUser($events->getUser()) - ->setEvent($events->getEvent()) - ->setParam('functionId', $events->getParam('functionId')) - ->setParam('executionId', $events->getParam('executionId')) + ->from($events) + ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + ->setClass(Event::FUNCTIONS_CLASS_NAME) ->trigger(); /** diff --git a/app/worker.php b/app/worker.php index 110a9d81da..60e93b2518 100644 --- a/app/worker.php +++ b/app/worker.php @@ -76,7 +76,7 @@ Server::setResource('functions', function (Registry $register) { ->get('queue') ->pop() ->getResource() - ); + ); }, ['register']); Server::setResource('logger', function ($register) { @@ -93,4 +93,4 @@ $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); $workerNumber = 1; -Runtime::enableCoroutine(SWOOLE_HOOK_ALL); \ No newline at end of file +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); diff --git a/app/workers/functions.php b/app/workers/functions.php index a34f0aec99..9d1dc0372d 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -52,37 +52,37 @@ Server::setResource('execute', function () { $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); var_dump("Deployment ID : ", $deploymentId); - + /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); - + if ($deployment->getAttribute('resourceId') !== $functionId) { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - + if ($deployment->isEmpty()) { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - + /** Check if build has exists */ $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($build->isEmpty()) { throw new Exception('Build not found'); } - + if ($build->getAttribute('status') !== 'ready') { throw new Exception('Build not ready'); } - + /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); - + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } - + $runtime = $runtimes[$function->getAttribute('runtime')]; - + /** Create execution or update execution status */ $execution = $dbForProject->getDocument('executions', $executionId ?? ''); if ($execution->isEmpty()) { @@ -100,27 +100,27 @@ Server::setResource('execute', function () { 'duration' => 0.0, 'search' => implode(' ', [$functionId, $executionId]), ])); - + if ($execution->isEmpty()) { throw new Exception('Failed to create or read execution'); } } $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - + if ($build->getAttribute('status') !== 'ready') { throw new Exception('Build not ready'); } - + /** Check if runtime is supported */ $runtimes = Config::getParam('runtimes', []); - + if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } - + $runtime = $runtimes[$function->getAttribute('runtime')]; - + /** Create execution or update execution status */ $execution = $dbForProject->getDocument('executions', $executionId ?? ''); if ($execution->isEmpty()) { @@ -138,19 +138,19 @@ Server::setResource('execute', function () { 'duration' => 0.0, 'search' => implode(' ', [$functionId, $executionId]), ])); - + if ($execution->isEmpty()) { throw new Exception('Failed to create or read execution'); } } $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - + $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value'); return $carry; }, []); - + /** Collect environment variables */ $vars = \array_merge($vars, [ 'APPWRITE_FUNCTION_ID' => $functionId, @@ -166,7 +166,7 @@ Server::setResource('execute', function () { 'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '', 'APPWRITE_FUNCTION_JWT' => $jwt ?? '', ]); - + /** Execute function */ $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { @@ -180,7 +180,7 @@ Server::setResource('execute', function () { source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); - + /** Update execution status */ $execution ->setAttribute('status', $executionResponse['status']) @@ -198,9 +198,9 @@ Server::setResource('execute', function () { ->setAttribute('stderr', $th->getMessage()); Console::error($th->getMessage()); } - + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - + /** Trigger Webhook */ $executionModel = new Execution(); $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); @@ -212,7 +212,7 @@ Server::setResource('execute', function () { ->setParam('executionId', $execution->getId()) ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) ->trigger(); - + /** Trigger Functions */ $functions ->setData($data ?? '') @@ -222,7 +222,7 @@ Server::setResource('execute', function () { ->setParam('functionId', $function->getId()) ->setParam('executionId', $execution->getId()) ->trigger(); - + /** Trigger realtime event */ $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ 'functionId' => $function->getId(), @@ -247,7 +247,7 @@ Server::setResource('execute', function () { channels: $target['channels'], roles: $target['roles'] ); - + /** Update usage stats */ if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $usage = new Stats($statsd); @@ -282,6 +282,7 @@ $server->job() $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; $data = $payload['data'] ?? ''; + $eventData = $payload['payload'] ?? ''; $project = new Document($payload['project'] ?? []); $function = new Document($payload['function'] ?? []); $user = new Document($payload['user'] ?? []); @@ -320,14 +321,14 @@ $server->job() $execute( statsd: $statsd, dbForProject: $dbForProject, - project: $project, + project: $project, function: $function, - trigger: 'event', - event: $events[0], - eventData: $payload, + trigger: 'event', + event: $events[0], + eventData: $eventData, user: $user, - data: null, - executionId: null, + data: null, + executionId: null, jwt: null ); Console::success('Triggered function: ' . $events[0]); @@ -346,15 +347,15 @@ $server->job() $execution = new Document($payload['execution'] ?? []); $user = new Document($payload['user'] ?? []); $execute( - project: $project, + project: $project, function: $function, dbForProject: $dbForProject, functions: $functions, trigger: 'http', - executionId: $execution->getId(), - event: null, + executionId: $execution->getId(), + event: null, eventData: null, - data: $data, + data: $data, user: $user, jwt: $jwt, statsd: $statsd, @@ -362,15 +363,15 @@ $server->job() break; case 'schedule': $execute( - project: $project, + project: $project, function: $function, dbForProject: $dbForProject, functions: $functions, trigger: 'http', - executionId: null, - event: null, + executionId: null, + event: null, eventData: null, - data: null, + data: null, user: null, jwt: null, statsd: $statsd, diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 514eb5df74..9fdcc4f8f3 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -145,16 +145,36 @@ class Func extends Event */ public function trigger(): string|bool { - $queue = new Client(Event::FUNCTIONS_QUEUE_NAME, $this->connection); + $client = new Client($this->queue, $this->connection); - return $queue->enqueue([ - 'type' => $this->type, - 'execution' => $this->execution, - 'function' => $this->function, - 'data' => $this->data, - 'jwt' => $this->jwt, + return $client->enqueue([ 'project' => $this->project, - 'user' => $this->user + 'user' => $this->user, + 'function' => $this->function, + 'execution' => $this->execution, + 'type' => $this->type, + 'jwt' => $this->jwt, + 'payload' => '', + 'events' => Event::generateEvents($this->getEvent(), $this->getParams()), + 'data' => $this->data, ]); } + + /** + * Generate a function event from a base event + * + * @param Event $event + * + * @return self + * + */ + public function from(Event $event): self + { + $this->project = $event->getProject(); + $this->user = $event->getUser(); + $this->payload = $event->getPayload(); + $this->event = $event->getEvent(); + $this->params = $event->getParams(); + return $this; + } } diff --git a/src/Appwrite/Utopia/Response/Model/Execution.php b/src/Appwrite/Utopia/Response/Model/Execution.php index 987a140dfb..13011a24b7 100644 --- a/src/Appwrite/Utopia/Response/Model/Execution.php +++ b/src/Appwrite/Utopia/Response/Model/Execution.php @@ -6,7 +6,6 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use Utopia\Database\Role; - class Execution extends Model { public function __construct() From e976bd3cb1ea022f0323ab2a46211dbaa5adcc64 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 11:22:19 +0530 Subject: [PATCH 17/31] fix: event triggers --- app/workers/functions.php | 2 -- src/Appwrite/Event/Event.php | 2 -- src/Appwrite/Event/Func.php | 4 +++- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 9d1dc0372d..e2a31afd75 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -51,7 +51,6 @@ Server::setResource('execute', function () { $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); - var_dump("Deployment ID : ", $deploymentId); /** Check if deployment exists */ $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -286,7 +285,6 @@ $server->job() $project = new Document($payload['project'] ?? []); $function = new Document($payload['function'] ?? []); $user = new Document($payload['user'] ?? []); - var_dump("Function : ", $function); if ($project->getId() === 'console') { return; diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 222cf59444..6e3401e11b 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -337,8 +337,6 @@ class Event default => false }; - - return [ 'type' => $type, 'resource' => $resource, diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 9fdcc4f8f3..b114ca4b59 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -147,6 +147,8 @@ class Func extends Event { $client = new Client($this->queue, $this->connection); + $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; + return $client->enqueue([ 'project' => $this->project, 'user' => $this->user, @@ -155,7 +157,7 @@ class Func extends Event 'type' => $this->type, 'jwt' => $this->jwt, 'payload' => '', - 'events' => Event::generateEvents($this->getEvent(), $this->getParams()), + 'events' => $events, 'data' => $this->data, ]); } From dbb05c46ba734bd438b291257cdeefdbf071eeb6 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 16 Nov 2022 11:38:01 +0530 Subject: [PATCH 18/31] fix: event triggers --- app/worker.php | 2 +- app/workers/functions.php | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/worker.php b/app/worker.php index 60e93b2518..c9074f6e20 100644 --- a/app/worker.php +++ b/app/worker.php @@ -69,7 +69,7 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); -Server::setResource('functions', function (Registry $register) { +Server::setResource('queueForFunctions', function (Registry $register) { $pools = $register->get('pools'); return new Func( $pools diff --git a/app/workers/functions.php b/app/workers/functions.php index e2a31afd75..bfa4421449 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -37,7 +37,7 @@ Server::setResource('execute', function () { Document $project, Document $function, Database $dbForProject, - Func $functions, + Func $queueForFunctions, string $trigger, string $executionId = null, string $event = null, @@ -213,7 +213,7 @@ Server::setResource('execute', function () { ->trigger(); /** Trigger Functions */ - $functions + $queueForFunctions ->setData($data ?? '') ->setProject($project) ->setUser($user) @@ -267,10 +267,10 @@ Server::setResource('execute', function () { $server->job() ->inject('message') ->inject('dbForProject') - ->inject('functions') + ->inject('queueForFunctions') ->inject('statsd') ->inject('execute') - ->action(function (Message $message, Database $dbForProject, Func $functions, Client $statsd, callable $execute) { + ->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute) { $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -321,6 +321,7 @@ $server->job() dbForProject: $dbForProject, project: $project, function: $function, + queueForFunctions: $queueForFunctions, trigger: 'event', event: $events[0], eventData: $eventData, @@ -348,7 +349,7 @@ $server->job() project: $project, function: $function, dbForProject: $dbForProject, - functions: $functions, + queueForFunctions: $queueForFunctions, trigger: 'http', executionId: $execution->getId(), event: null, @@ -364,7 +365,7 @@ $server->job() project: $project, function: $function, dbForProject: $dbForProject, - functions: $functions, + queueForFunctions: $queueForFunctions, trigger: 'http', executionId: null, event: null, From 4e3bcd6cd0dbb43308187bf973519ab9d335f40a Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 10:41:57 +0200 Subject: [PATCH 19/31] Authorization::skip added --- app/controllers/api/functions.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b695d19429..7c9bd4feaf 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -484,7 +484,7 @@ App::put('/v1/functions/:functionId') ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $eventsInstance->setParam('functionId', $function->getId()); @@ -1157,8 +1157,7 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1218,10 +1217,10 @@ App::post('/v1/functions/:functionId/executions') // TODO revise this later using route label $usage - ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')); // ms + ->setParam('functionId', $function->getId()) + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('duration')); // ms $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); From 6b88f66d14939b9ad061709611124287512e3aae Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 10:46:47 +0200 Subject: [PATCH 20/31] removed pools injection in functions controller --- app/controllers/api/functions.php | 4 +--- app/init.php | 17 ----------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7c9bd4feaf..d64f302636 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -14,7 +14,6 @@ use Utopia\Database\Permission; use Utopia\Database\Role; use Utopia\Database\Validator\UID; use Appwrite\Usage\Stats; -use Utopia\Pools\Group; use Utopia\Storage\Device; use Utopia\Storage\Validator\File; use Utopia\Storage\Validator\FileExt; @@ -1060,9 +1059,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('events') ->inject('usage') ->inject('mode') - ->inject('pools') ->inject('functions') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Group $pools, Func $functions) { + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $functions) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); diff --git a/app/init.php b/app/init.php index 19cea371df..a6c4b89c45 100644 --- a/app/init.php +++ b/app/init.php @@ -1026,23 +1026,6 @@ App::setResource('console', function () { ]); }, []); -App::setResource('queue', function () { - $fallbackForRedis = AppwriteURL::unparse([ - 'scheme' => 'redis', - 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), - 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), - 'user' => App::getEnv('_APP_REDIS_USER', ''), - 'pass' => App::getEnv('_APP_REDIS_PASS', ''), - ]); - - $connection = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); - - $dsns = explode(',', $connection ?? ''); - $dsn = explode('=', $dsns[0]); - $dsn = $dsn[1] ?? ''; - return new DSN($dsn); -}, []); - App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; From ac70537ece6730d7359e4cdc1ad60971fda29942 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 10:50:24 +0200 Subject: [PATCH 21/31] resetting workers-number --- app/worker.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/worker.php b/app/worker.php index c9074f6e20..5949bd936f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -91,6 +91,5 @@ $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -$workerNumber = 1; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); From 0d61a7fcf5a6511b82b705703ea496c937cc06b2 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 16 Nov 2022 11:00:11 +0200 Subject: [PATCH 22/31] resetting workers-number --- app/worker.php | 1 - docker-compose.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/app/worker.php b/app/worker.php index 5949bd936f..3cdfb81aab 100644 --- a/app/worker.php +++ b/app/worker.php @@ -89,7 +89,6 @@ Server::setResource('statsd', function ($register) { $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); - $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); Runtime::enableCoroutine(SWOOLE_HOOK_ALL); diff --git a/docker-compose.yml b/docker-compose.yml index 4bef6b1a01..22ed89bed4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -500,6 +500,7 @@ services: - _APP_USAGE_STATS - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD + - _APP_WORKER_PER_CORE appwrite-worker-mails: entrypoint: worker-mails From 44875e2d18b44804ceee8814aaaac1be4184835f Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 11:33:11 +0100 Subject: [PATCH 23/31] QA review changes --- app/cli.php | 2 +- app/controllers/api/functions.php | 18 +++++++++--------- app/controllers/shared/api.php | 8 +++----- app/init.php | 2 +- app/workers/builds.php | 22 +++++++++++++--------- app/workers/functions.php | 3 ++- src/Appwrite/Platform/Tasks/Schedule.php | 4 +--- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/app/cli.php b/app/cli.php index 544148c8d5..13709b9b57 100644 --- a/app/cli.php +++ b/app/cli.php @@ -110,7 +110,7 @@ CLI::setResource('influxdb', function (Registry $register) { return $database; }, ['register']); -CLI::setResource('functions', function (Group $pools) { +CLI::setResource('queueForFunctions', function (Group $pools) { return new Func($pools->get('queue')->pop()->getResource()); }, ['pools']); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d64f302636..86981b0071 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -469,13 +469,13 @@ App::put('/v1/functions/:functionId') 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); /** * In case we want to clear the schedule */ if (!empty($function->getAttribute('deployment'))) { - $schedule->setAttribute('resourceUpdatedAt', $function['scheduleUpdatedAt']); + $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); } $schedule @@ -537,7 +537,7 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') 'deployment' => $deployment->getId() ]))); - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $active = !empty($function->getAttribute('schedule')); @@ -588,7 +588,7 @@ App::delete('/v1/functions/:functionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB'); } - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) @@ -790,7 +790,7 @@ App::post('/v1/functions/:functionId/deployments') * TODO Should we update also the function collection with the scheduleUpdatedAt attr? */ - $schedule = $dbForConsole->getDocument('schedules', $function['scheduleId']); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $active = !empty($function->getAttribute('schedule')); @@ -1059,8 +1059,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('events') ->inject('usage') ->inject('mode') - ->inject('functions') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $functions) { + ->inject('queueForFunctions') + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $queueForFunctions) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1148,7 +1148,7 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - $functions + $queueForFunctions ->setType('http') ->setExecution($execution) ->setFunction($function) @@ -1162,7 +1162,7 @@ App::post('/v1/functions/:functionId/executions') ->dynamic($execution, Response::MODEL_EXECUTION); } - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value') ?? ''; return $carry; }, []); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a12016606f..5624486755 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -252,8 +252,8 @@ App::shutdown() ->inject('database') ->inject('mode') ->inject('dbForProject') - ->inject('functions') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $functions) use ($parseLabel) { + ->inject('queueForFunctions') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $queueForFunctions) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -264,10 +264,8 @@ App::shutdown() /** * Trigger functions. */ - $functions + $queueForFunctions ->from($events) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) - ->setClass(Event::FUNCTIONS_CLASS_NAME) ->trigger(); /** diff --git a/app/init.php b/app/init.php index a6c4b89c45..af3b05965d 100644 --- a/app/init.php +++ b/app/init.php @@ -844,7 +844,7 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); -App::setResource('functions', function (Group $pools) { +App::setResource('queueForFunctions', function (Group $pools) { return new Func($pools->get('queue')->pop()->getResource()); }, ['pools']); App::setResource('usage', function ($register) { diff --git a/app/workers/builds.php b/app/workers/builds.php index 315b13a179..478c5c5423 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -15,6 +15,7 @@ use Utopia\Database\ID; use Utopia\Storage\Storage; use Utopia\Database\Document; use Utopia\Config\Config; +use Utopia\Database\Validator\Authorization; require_once __DIR__ . '/../init.php'; @@ -156,7 +157,7 @@ class BuildsV1 extends Worker $source = $deployment->getAttribute('path'); - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value'); return $carry; }, []); @@ -192,6 +193,8 @@ class BuildsV1 extends Worker Console::success("Build id: $buildId created"); + $function->setAttribute('scheduleUpdatedAt', DateTime::now()); + /** Set auto deploy */ if ($deployment->getAttribute('activate') === true) { $function->setAttribute('deployment', $deployment->getId()); @@ -199,14 +202,15 @@ class BuildsV1 extends Worker } /** Update function schedule */ - // TODO: @Meldiron refactor scheduler here - /* - $schedule = $function->getAttribute('schedule', ''); - $cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null; - $next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? DateTime::format($cron->getNextRunDate()) : null; - $function->setAttribute('scheduleNext', $next); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - */ + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); + + $schedule + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + + + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); diff --git a/app/workers/functions.php b/app/workers/functions.php index bfa4421449..68b9c21bf6 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -29,6 +29,7 @@ Authorization::setDefaultStatus(false); global $connection; global $workerNumber; + $adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); $server = new Server($adapter); @@ -145,7 +146,7 @@ Server::setResource('execute', function () { $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - $vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) { + $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { $carry[$var->getAttribute('key')] = $var->getAttribute('value'); return $carry; }, []); diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index 88fd0a9a80..dc86f9293e 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -3,17 +3,15 @@ namespace Appwrite\Platform\Tasks; use Cron\CronExpression; +use Swoole\Timer; use Utopia\App; use Utopia\Platform\Action; use Utopia\CLI\Console; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Query; -use Swoole\Timer; use Utopia\Database\Database; use Utopia\Pools\Group; -use Utopia\Queue\Client as Worker; -use Appwrite\Event\Event; use Appwrite\Event\Func; use function Swoole\Coroutine\run; From 9e4a65605c55b00015da94103cb8985028190efc Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 11:40:41 +0100 Subject: [PATCH 24/31] Simplify worker --- app/worker.php | 56 ++++++++++++++++++++++++++++++++++++++- app/workers/functions.php | 52 ------------------------------------ bin/worker-functions | 2 +- composer.json | 2 +- 4 files changed, 57 insertions(+), 55 deletions(-) diff --git a/app/worker.php b/app/worker.php index 3cdfb81aab..7bc1139251 100644 --- a/app/worker.php +++ b/app/worker.php @@ -7,12 +7,17 @@ use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; +use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Queue\Adapter\Swoole; use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; +use Utopia\Logger\Log; + +Runtime::enableCoroutine(SWOOLE_HOOK_ALL); global $register; @@ -91,4 +96,53 @@ $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -Runtime::enableCoroutine(SWOOLE_HOOK_ALL); +if(empty(App::getEnv('QUEUE'))) { + throw new Exception('Please configure "QUEUE" environemnt variable.'); +} + +$adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE')); +$server = new Server($adapter); + +$server + ->error() + ->inject('error') + ->inject('logger') + ->inject('register') + ->action(function ($error, $logger, $register) { + + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); + + if ($error instanceof PDOException) { + throw $error; + } + + if ($error->getCode() >= 500 || $error->getCode() === 0) { + $log = new Log(); + + $log->setNamespace("appwrite-worker"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + $log->setAction('appwrite-worker-functions'); + $log->addTag('verboseType', get_class($error)); + $log->addTag('code', $error->getCode()); + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + $log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $logger->addLog($log); + } + + Console::error('[Error] Type: ' . get_class($error)); + Console::error('[Error] Message: ' . $error->getMessage()); + Console::error('[Error] File: ' . $error->getFile()); + Console::error('[Error] Line: ' . $error->getLine()); + + $register->get('pools')->reclaim(); + }); \ No newline at end of file diff --git a/app/workers/functions.php b/app/workers/functions.php index 68b9c21bf6..f1e472b471 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -20,19 +20,11 @@ use Utopia\Database\Permission; use Utopia\Database\Query; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; -use Utopia\Logger\Log; -use Utopia\Queue\Adapter\Swoole; use Utopia\Queue\Server; Authorization::disable(); Authorization::setDefaultStatus(false); -global $connection; -global $workerNumber; - -$adapter = new Swoole($connection, $workerNumber, Event::FUNCTIONS_QUEUE_NAME); -$server = new Server($adapter); - Server::setResource('execute', function () { return function ( Document $project, @@ -380,49 +372,5 @@ $server->job() } }); -$server - ->error() - ->inject('error') - ->inject('logger') - ->inject('register') - ->action(function ($error, $logger, $register) { - - $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - - if ($error instanceof PDOException) { - throw $error; - } - - if ($error->getCode() >= 500 || $error->getCode() === 0) { - $log = new Log(); - - $log->setNamespace("appwrite-worker"); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - $log->setAction('appwrite-worker-functions'); - $log->addTag('verboseType', get_class($error)); - $log->addTag('code', $error->getCode()); - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - $log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - $logger->addLog($log); - } - - Console::error('[Error] Type: ' . get_class($error)); - Console::error('[Error] Message: ' . $error->getMessage()); - Console::error('[Error] File: ' . $error->getFile()); - Console::error('[Error] Line: ' . $error->getLine()); - - $register->get('pools')->reclaim(); - }); - $server->workerStart(); $server->start(); diff --git a/bin/worker-functions b/bin/worker-functions index b22ee65d39..687f9fd0cd 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file +QUEUE=v1-functions php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 08ede65dcf..ffd9c6b6b9 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/platform": "0.3.*", "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", - "utopia-php/registry": "dev-feat-allow-params as 0.5.0", + "utopia-php/registry": "0.5.*", "utopia-php/storage": "0.11.*", "utopia-php/swoole": "0.5.*", "utopia-php/websocket": "0.1.0", From 390f407eb5cda77e6595c3bb4183a5babf0bb9ac Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 11:54:21 +0100 Subject: [PATCH 25/31] Linter fixes --- app/worker.php | 4 ++-- app/workers/builds.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/worker.php b/app/worker.php index 7bc1139251..eb7e570007 100644 --- a/app/worker.php +++ b/app/worker.php @@ -96,7 +96,7 @@ $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -if(empty(App::getEnv('QUEUE'))) { +if (empty(App::getEnv('QUEUE'))) { throw new Exception('Please configure "QUEUE" environemnt variable.'); } @@ -145,4 +145,4 @@ $server Console::error('[Error] Line: ' . $error->getLine()); $register->get('pools')->reclaim(); - }); \ No newline at end of file + }); diff --git a/app/workers/builds.php b/app/workers/builds.php index 478c5c5423..ed1c16e8bf 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -208,8 +208,8 @@ class BuildsV1 extends Worker $schedule ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - - + + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); } catch (\Throwable $th) { $endTime = DateTime::now(); From 43adf758bd88f9a857e221f6d11bd98c55f4a8cb Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 12:30:57 +0100 Subject: [PATCH 26/31] Fix lockfile after merge --- composer.lock | 65 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/composer.lock b/composer.lock index 2e39916a0c..517065af56 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "bf3e2ed6ee8e49ab74af97b368b89a63", + "content-hash": "a673091aa6bd8ef01380b63245427c93", "packages": [ { "name": "adhocore/jwt", @@ -2357,6 +2357,67 @@ }, "time": "2020-10-24T07:04:59+00:00" }, + { + "name": "utopia-php/queue", + "version": "0.4.1", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/queue.git", + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.14.*", + "utopia-php/framework": "0.*.*" + }, + "require-dev": { + "laravel/pint": "^0.2.3", + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.5.5", + "swoole/ide-helper": "4.8.8", + "workerman/workerman": "^4.0" + }, + "suggest": { + "ext-swoole": "Needed to support Swoole.", + "workerman/workerman": "Needed to support Workerman." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Queue\\": "src/Queue" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A powerful task queue.", + "keywords": [ + "Tasks", + "framework", + "php", + "queue", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/queue/issues", + "source": "https://github.com/utopia-php/queue/tree/0.4.1" + }, + "time": "2022-11-15T16:56:37+00:00" + }, { "name": "utopia-php/registry", "version": "0.5.0", @@ -5241,5 +5302,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } From fdbb5b9810351e37279fce85778849847b9d8a4a Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 13:19:29 +0100 Subject: [PATCH 27/31] QA bug fixing --- app/workers/builds.php | 6 +- app/workers/functions.php | 95 +++++++++++------------- docker-compose.yml | 14 +++- src/Appwrite/Platform/Tasks/Schedule.php | 13 ++-- 4 files changed, 69 insertions(+), 59 deletions(-) diff --git a/app/workers/builds.php b/app/workers/builds.php index ed1c16e8bf..48d430eaa7 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -5,7 +5,6 @@ use Appwrite\Event\Func; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Resque\Worker; use Appwrite\Utopia\Response\Model\Deployment; -use Cron\CronExpression; use Executor\Executor; use Appwrite\Usage\Stats; use Utopia\Database\DateTime; @@ -59,6 +58,8 @@ class BuildsV1 extends Worker protected function buildDeployment(Document $project, Document $function, Document $deployment) { + global $register; + $dbForProject = $this->getProjectDB($project); $function = $dbForProject->getDocument('functions', $function->getId()); @@ -122,7 +123,6 @@ class BuildsV1 extends Worker ->trigger(); /** Trigger Functions */ - global $register; $pools = $register->get('pools'); $connection = $pools->get('queue')->pop(); $functions = new Func($connection->getResource()); @@ -202,6 +202,7 @@ class BuildsV1 extends Worker } /** Update function schedule */ + $dbForConsole = $this->getConsoleDB(); $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); @@ -240,7 +241,6 @@ class BuildsV1 extends Worker ); /** Update usage stats */ - global $register; if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $statsd = $register->get('statsd'); $usage = new Stats($statsd); diff --git a/app/workers/functions.php b/app/workers/functions.php index f1e472b471..906ae25d38 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -270,7 +270,6 @@ $server->job() throw new Exception('Missing payload'); } - var_dump(json_encode($payload)); $type = $payload['type'] ?? ''; $events = $payload['events'] ?? []; $data = $payload['data'] ?? ''; @@ -283,53 +282,6 @@ $server->job() return; } - /** - * Handle Event execution. - */ - if (!empty($events)) { - $limit = 30; - $sum = 30; - $offset = 0; - $functions = []; - /** @var Document[] $functions */ - while ($sum >= $limit) { - $functions = $dbForProject->find('functions', [ - Query::limit($limit), - Query::offset($offset), - Query::orderAsc('name'), - ]); - - $sum = \count($functions); - $offset = $offset + $limit; - - Console::log('Fetched ' . $sum . ' functions...'); - - foreach ($functions as $function) { - if (!array_intersect($events, $function->getAttribute('events', []))) { - continue; - } - Console::success('Iterating function: ' . $function->getAttribute('name')); - $execute( - statsd: $statsd, - dbForProject: $dbForProject, - project: $project, - function: $function, - queueForFunctions: $queueForFunctions, - trigger: 'event', - event: $events[0], - eventData: $eventData, - user: $user, - data: null, - executionId: null, - jwt: null - ); - Console::success('Triggered function: ' . $events[0]); - } - } - - return; - } - /** * Handle Schedule and HTTP execution. */ @@ -343,7 +295,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: 'http', + trigger: $type, executionId: $execution->getId(), event: null, eventData: null, @@ -359,7 +311,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: 'http', + trigger: $type, executionId: null, event: null, eventData: null, @@ -369,6 +321,49 @@ $server->job() statsd: $statsd, ); break; + case 'event': + if (!empty($events)) { + $limit = 30; + $sum = 30; + $offset = 0; + $functions = []; + /** @var Document[] $functions */ + while ($sum >= $limit) { + $functions = $dbForProject->find('functions', [ + Query::limit($limit), + Query::offset($offset), + Query::orderAsc('name'), + ]); + + $sum = \count($functions); + $offset = $offset + $limit; + + Console::log('Fetched ' . $sum . ' functions...'); + + foreach ($functions as $function) { + if (!array_intersect($events, $function->getAttribute('events', []))) { + continue; + } + Console::success('Iterating function: ' . $function->getAttribute('name')); + $execute( + statsd: $statsd, + dbForProject: $dbForProject, + project: $project, + function: $function, + queueForFunctions: $queueForFunctions, + trigger: $type, + event: $events[0], + eventData: $eventData, + user: $user, + data: null, + executionId: null, + jwt: null + ); + Console::success('Triggered function: ' . $events[0]); + } + } + } + break; } }); diff --git a/docker-compose.yml b/docker-compose.yml index 7b69254738..d7fe33d536 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -258,6 +258,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -291,6 +292,7 @@ services: - request-catcher environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -321,6 +323,7 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -376,6 +379,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -408,6 +412,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -444,6 +449,7 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET @@ -480,6 +486,7 @@ services: - openruntimes-executor environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -500,7 +507,6 @@ services: - _APP_USAGE_STATS - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD - - _APP_WORKER_PER_CORE appwrite-worker-mails: entrypoint: worker-mails @@ -518,6 +524,7 @@ services: # - smtp environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS @@ -548,6 +555,7 @@ services: - redis environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -572,6 +580,7 @@ services: - redis environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 @@ -624,6 +633,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -662,6 +672,7 @@ services: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -697,6 +708,7 @@ services: - redis environment: - _APP_ENV + - _APP_WORKER_PER_CORE - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index dc86f9293e..bafedd27cb 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -97,7 +97,7 @@ class Schedule extends Action $pools->reclaim(); - Console::success("{$total} functions where loaded in " . (microtime(true) - $loadStart) . " seconds"); + Console::success("{$total} functions were loaded in " . (microtime(true) - $loadStart) . " seconds"); Console::success("Starting timers at " . DateTime::now()); @@ -152,7 +152,7 @@ class Schedule extends Action $pools->reclaim(); - Console::log("Sync tick: {$total} schedules where updates in " . ($timerEnd - $timerStart) . " seconds"); + Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds"); }); /** @@ -185,11 +185,14 @@ class Schedule extends Action $total++; - $promiseStart = \microtime(true); // in seconds + $promiseStart = \time(); // in seconds $executionStart = $nextDate->getTimestamp(); // in seconds $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued + $delay = $executionSleep; - $delay = \ceil(\intval($executionSleep)); + \var_dump($delay); + \var_dump(\time()); + \var_dump('---'); if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; @@ -228,7 +231,7 @@ class Schedule extends Action $timerEnd = \microtime(true); $lastEnqueueUpdate = $timerStart; - Console::log("Enqueue tick: {$total} executions where enqueued in " . ($timerEnd - $timerStart) . " seconds"); + Console::log("Enqueue tick: {$total} executions were enqueued in " . ($timerEnd - $timerStart) . " seconds"); }; Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions()); From 3e325afb616b57ce688b08514f6000a289bdb9a0 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 13:44:14 +0100 Subject: [PATCH 28/31] Revert unnessessary changes --- app/controllers/api/functions.php | 3 +- app/workers/functions.php | 85 +++++++++++++++---------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 86981b0071..a94cde197d 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1155,7 +1155,8 @@ App::post('/v1/functions/:functionId/executions') ->setData($data) ->setJWT($jwt) ->setProject($project) - ->setUser($user); + ->setUser($user) + ->trigger(); return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/workers/functions.php b/app/workers/functions.php index 906ae25d38..d503fdc5e0 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -282,6 +282,48 @@ $server->job() return; } + if (!empty($events)) { + $limit = 30; + $sum = 30; + $offset = 0; + $functions = []; + /** @var Document[] $functions */ + while ($sum >= $limit) { + $functions = $dbForProject->find('functions', [ + Query::limit($limit), + Query::offset($offset), + Query::orderAsc('name'), + ]); + + $sum = \count($functions); + $offset = $offset + $limit; + + Console::log('Fetched ' . $sum . ' functions...'); + + foreach ($functions as $function) { + if (!array_intersect($events, $function->getAttribute('events', []))) { + continue; + } + Console::success('Iterating function: ' . $function->getAttribute('name')); + $execute( + statsd: $statsd, + dbForProject: $dbForProject, + project: $project, + function: $function, + queueForFunctions: $queueForFunctions, + trigger: $type, + event: $events[0], + eventData: $eventData, + user: $user, + data: null, + executionId: null, + jwt: null + ); + Console::success('Triggered function: ' . $events[0]); + } + } + } + /** * Handle Schedule and HTTP execution. */ @@ -321,49 +363,6 @@ $server->job() statsd: $statsd, ); break; - case 'event': - if (!empty($events)) { - $limit = 30; - $sum = 30; - $offset = 0; - $functions = []; - /** @var Document[] $functions */ - while ($sum >= $limit) { - $functions = $dbForProject->find('functions', [ - Query::limit($limit), - Query::offset($offset), - Query::orderAsc('name'), - ]); - - $sum = \count($functions); - $offset = $offset + $limit; - - Console::log('Fetched ' . $sum . ' functions...'); - - foreach ($functions as $function) { - if (!array_intersect($events, $function->getAttribute('events', []))) { - continue; - } - Console::success('Iterating function: ' . $function->getAttribute('name')); - $execute( - statsd: $statsd, - dbForProject: $dbForProject, - project: $project, - function: $function, - queueForFunctions: $queueForFunctions, - trigger: $type, - event: $events[0], - eventData: $eventData, - user: $user, - data: null, - executionId: null, - jwt: null - ); - Console::success('Triggered function: ' . $events[0]); - } - } - } - break; } }); From 3b6174b5a7bdba2a5ddeb6c9235af2fd4dbf5c80 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 14:11:00 +0100 Subject: [PATCH 29/31] Bug fixes --- app/workers/functions.php | 47 +++++---------------------------------- 1 file changed, 6 insertions(+), 41 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index d503fdc5e0..209c9a31df 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -40,7 +40,6 @@ Server::setResource('execute', function () { string $jwt = null, Client $statsd ) { - $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); @@ -93,48 +92,13 @@ Server::setResource('execute', function () { 'search' => implode(' ', [$functionId, $executionId]), ])); - if ($execution->isEmpty()) { - throw new Exception('Failed to create or read execution'); - } - } - $execution->setAttribute('status', 'processing'); - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - - if ($build->getAttribute('status') !== 'ready') { - throw new Exception('Build not ready'); - } - - /** Check if runtime is supported */ - $runtimes = Config::getParam('runtimes', []); - - if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - $runtime = $runtimes[$function->getAttribute('runtime')]; - - /** Create execution or update execution status */ - $execution = $dbForProject->getDocument('executions', $executionId ?? ''); - if ($execution->isEmpty()) { - $executionId = ID::unique(); - $execution = $dbForProject->createDocument('executions', new Document([ - '$id' => $executionId, - '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))], - 'functionId' => $functionId, - 'deploymentId' => $deploymentId, - 'trigger' => $trigger, - 'status' => 'waiting', - 'statusCode' => 0, - 'response' => '', - 'stderr' => '', - 'duration' => 0.0, - 'search' => implode(' ', [$functionId, $executionId]), - ])); + // TODO: @Meldiron Trigger executions.create event here if ($execution->isEmpty()) { throw new Exception('Failed to create or read execution'); } } + $execution->setAttribute('status', 'processing'); $execution = $dbForProject->updateDocument('executions', $executionId, $execution); @@ -311,7 +275,7 @@ $server->job() project: $project, function: $function, queueForFunctions: $queueForFunctions, - trigger: $type, + trigger: 'event', event: $events[0], eventData: $eventData, user: $user, @@ -322,6 +286,7 @@ $server->job() Console::success('Triggered function: ' . $events[0]); } } + return; } /** @@ -337,7 +302,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: $type, + trigger: 'http', executionId: $execution->getId(), event: null, eventData: null, @@ -353,7 +318,7 @@ $server->job() function: $function, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - trigger: $type, + trigger: 'schedule', executionId: null, event: null, eventData: null, From e37f70246e4865434eefc6355476a89433a02d21 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 14:34:11 +0100 Subject: [PATCH 30/31] PR review changes --- app/init.php | 1 - app/worker.php | 9 +++------ app/workers/builds.php | 10 ++-------- src/Appwrite/Platform/Tasks/Schedule.php | 7 +------ 4 files changed, 6 insertions(+), 21 deletions(-) diff --git a/app/init.php b/app/init.php index 067fd70204..31f82740b6 100644 --- a/app/init.php +++ b/app/init.php @@ -565,7 +565,6 @@ $register->set('pools', function () { $dsns = $connection['dsns'] ?? ''; $multipe = $connection['multiple'] ?? false; $schemes = $connection['schemes'] ?? []; - $useResource = $connection['useResource'] ?? true; $config = []; $dsns = explode(',', $connection['dsns'] ?? ''); diff --git a/app/worker.php b/app/worker.php index eb7e570007..42a5f92439 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,6 +16,7 @@ use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Logger\Log; +use Utopia\Logger\Logger; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); @@ -107,9 +108,7 @@ $server ->error() ->inject('error') ->inject('logger') - ->inject('register') - ->action(function ($error, $logger, $register) { - + ->action(function (Throwable $error, Logger $logger) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); if ($error instanceof PDOException) { @@ -124,7 +123,7 @@ $server $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); - $log->setAction('appwrite-worker-functions'); + $log->setAction('appwrite-queue-' . App::getEnv('QUEUE')); $log->addTag('verboseType', get_class($error)); $log->addTag('code', $error->getCode()); $log->addExtra('file', $error->getFile()); @@ -143,6 +142,4 @@ $server Console::error('[Error] Message: ' . $error->getMessage()); Console::error('[Error] File: ' . $error->getFile()); Console::error('[Error] Line: ' . $error->getLine()); - - $register->get('pools')->reclaim(); }); diff --git a/app/workers/builds.php b/app/workers/builds.php index 48d430eaa7..1fa86b99a8 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -111,15 +111,13 @@ class BuildsV1 extends Worker /** Trigger Webhook */ $deploymentModel = new Deployment(); - $data = $deployment->getArrayCopy(array_keys($deploymentModel->getRules())); - $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); $deploymentUpdate ->setProject($project) ->setEvent('functions.[functionId].deployments.[deploymentId].update') ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) - ->setPayload($data) + ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) ->trigger(); /** Trigger Functions */ @@ -127,11 +125,7 @@ class BuildsV1 extends Worker $connection = $pools->get('queue')->pop(); $functions = new Func($connection->getResource()); $functions - ->setData(\json_encode($data)) - ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()) + ->from($deploymentUpdate) ->trigger(); $connection->reclaim(); diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index bafedd27cb..1d89d85a04 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -187,12 +187,7 @@ class Schedule extends Action $promiseStart = \time(); // in seconds $executionStart = $nextDate->getTimestamp(); // in seconds - $executionSleep = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued - $delay = $executionSleep; - - \var_dump($delay); - \var_dump(\time()); - \var_dump('---'); + $delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; From 5f8d3ff6ee8a4645bc1613b144b5f1cb45badd1f Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 14:50:12 +0100 Subject: [PATCH 31/31] More PR reviews --- app/workers/functions.php | 25 ++++++++++--------------- src/Appwrite/Event/Func.php | 2 +- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 209c9a31df..3047bf027e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -27,18 +27,18 @@ Authorization::setDefaultStatus(false); Server::setResource('execute', function () { return function ( + Func $queueForFunctions, + Database $dbForProject, + Client $statsd, Document $project, Document $function, - Database $dbForProject, - Func $queueForFunctions, string $trigger, - string $executionId = null, - string $event = null, - string $eventData = null, string $data = null, ?Document $user = null, string $jwt = null, - Client $statsd + string $event = null, + string $eventData = null, + string $executionId = null, ) { $user ??= new Document(); $functionId = $function->getId(); @@ -124,11 +124,11 @@ Server::setResource('execute', function () { ]); /** Execute function */ - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { - $executionResponse = $executor->createExecution( + $client = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + $executionResponse = $client->createExecution( projectId: $project->getId(), - deploymentId: $deployment->getId(), + deploymentId: $deploymentId, payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '', variables: $vars, timeout: $function->getAttribute('timeout', 0), @@ -171,12 +171,7 @@ Server::setResource('execute', function () { /** Trigger Functions */ $queueForFunctions - ->setData($data ?? '') - ->setProject($project) - ->setUser($user) - ->setEvent('functions.[functionId].executions.[executionId].update') - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) + ->from($executionUpdate) ->trigger(); /** Trigger realtime event */ diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index b114ca4b59..22940ad08e 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -156,7 +156,7 @@ class Func extends Event 'execution' => $this->execution, 'type' => $this->type, 'jwt' => $this->jwt, - 'payload' => '', + 'payload' => $this->payload, 'events' => $events, 'data' => $this->data, ]);