From 033bbec261b5477ec9fbe979e26c770e0520ffb5 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 19 Apr 2022 15:13:55 +0200 Subject: [PATCH] fix: usage and build --- app/controllers/api/functions.php | 66 ++++++++++--------- app/tasks/maintenance.php | 54 +++++++-------- app/workers/builds.php | 57 ++++++++-------- src/Appwrite/Event/Build.php | 105 ++++++++++++++++++++++++++++++ src/Appwrite/Event/Delete.php | 27 ++++++++ 5 files changed, 215 insertions(+), 94 deletions(-) create mode 100644 src/Appwrite/Event/Build.php diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b8038676ba..da9232dabb 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -2,7 +2,7 @@ use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; -use Appwrite\Event\Event; +use Appwrite\Event\Build; use Appwrite\Event\Func; use Appwrite\Event\Validator\Event as ValidatorEvent; use Appwrite\Extend\Exception; @@ -313,7 +313,7 @@ App::put('/v1/functions/:functionId') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForProject */ /** @var Utopia\Database\Document $project */ - /** @var Appwrite\Auth\User $user */ + /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $eventsInstance */ $function = $dbForProject->getDocument('functions', $functionId); @@ -339,14 +339,15 @@ App::put('/v1/functions/:functionId') ]))); if ($next && $schedule !== $original) { - ResqueScheduler::enqueueAt($next, Event::FUNCTIONS_QUEUE_NAME, Event::FUNCTIONS_CLASS_NAME, [ - 'projectId' => $project->getId(), - 'webhooks' => $project->getAttribute('webhooks', []), - 'functionId' => $function->getId(), - 'userId' => $user->getId(), - 'executionId' => null, - 'trigger' => 'schedule', - ]); // Async task rescheduale + // Async task reschedule + $functionEvent = new Func(); + $functionEvent + ->setFunction($function) + ->setType('schedule') + ->setUser($user) + ->setProject($project); + + $functionEvent->schedule($next); } $eventsInstance->setParam('functionId', $function->getId()); @@ -408,13 +409,12 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') ]))); if ($next) { // Init first schedule - ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [ - 'projectId' => $project->getId(), - 'webhooks' => $project->getAttribute('webhooks', []), - 'functionId' => $function->getId(), - 'executionId' => null, - 'trigger' => 'schedule', - ]); // Async task rescheduale + $functionEvent = new Func(); + $functionEvent + ->setType('schedule') + ->setFunction($function) + ->setProject($project); + $functionEvent->schedule($next); } $events @@ -497,7 +497,7 @@ App::post('/v1/functions/:functionId/deployments') /** @var Utopia\Database\Database $dbForProject */ /** @var Appwrite\Event\Event $usage */ /** @var Appwrite\Event\Event $events */ - /** @var Appwrite\Database\Document $project */ + /** @var Utopia\Database\Document $project */ /** @var Utopia\Storage\Device $deviceFunctions */ /** @var Utopia\Storage\Device $deviceLocal */ @@ -616,13 +616,14 @@ App::post('/v1/functions/:functionId/deployments') $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('size', $fileSize)->setAttribute('metadata', $metadata)); } - // Enqueue a message to start the build - Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ - 'projectId' => $project->getId(), - 'resourceId' => $function->getId(), - 'deploymentId' => $deploymentId, - 'type' => BUILD_TYPE_DEPLOYMENT - ]); + // Start the build + $buildEvent = new Build(); + $buildEvent + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($function) + ->setDeployment($deployment) + ->setProject($project) + ->trigger(); $usage->setParam('storage', $deployment->getAttribute('size', 0)); } else { @@ -1150,13 +1151,14 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); - // Enqueue a message to start the build - Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ - 'projectId' => $project->getId(), - 'resourceId' => $function->getId(), - 'deploymentId' => $deploymentId, - 'type' => BUILD_TYPE_RETRY - ]); + // Retry the build + $buildEvent = new Build(); + $buildEvent + ->setType(BUILD_TYPE_RETRY) + ->setResource($function) + ->setDeployment($deployment) + ->setProject($project) + ->trigger(); $response->noContent(); }); diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 05a9607883..b53628e699 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -2,7 +2,7 @@ global $cli; -use Appwrite\Event\Event; +use Appwrite\Event\Delete; use Utopia\App; use Utopia\CLI\Console; @@ -15,53 +15,43 @@ $cli function notifyDeleteExecutionLogs(int $interval) { - Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'payload' => [ - 'type' => DELETE_TYPE_EXECUTIONS, - 'timestamp' => time() - $interval - ] - ]); + (new Delete()) + ->setType(DELETE_TYPE_EXECUTIONS) + ->setTimestamp(time() - $interval) + ->trigger(); } function notifyDeleteAbuseLogs(int $interval) { - Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'payload' => [ - 'type' => DELETE_TYPE_ABUSE, - 'timestamp' => time() - $interval - ] - ]); + (new Delete()) + ->setType(DELETE_TYPE_ABUSE) + ->setTimestamp(time() - $interval) + ->trigger(); } function notifyDeleteAuditLogs(int $interval) { - Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'payload' => [ - 'type' => DELETE_TYPE_AUDIT, - 'timestamp' => time() - $interval - ] - ]); + (new Delete()) + ->setType(DELETE_TYPE_AUDIT) + ->setTimestamp(time() - $interval) + ->trigger(); } function notifyDeleteUsageStats(int $interval30m, int $interval1d) { - Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'payload' => [ - 'type' => DELETE_TYPE_USAGE, - 'timestamp1d' => time() - $interval1d, - 'timestamp30m' => time() - $interval30m, - ] - ]); + (new Delete()) + ->setType(DELETE_TYPE_USAGE) + ->setTimestamp1d(time() - $interval1d) + ->setTimestamp30m(time() - $interval30m) + ->trigger(); } function notifyDeleteConnections() { - Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'payload' => [ - 'type' => DELETE_TYPE_REALTIME, - 'timestamp' => time() - 60 - ] - ]); + (new Delete()) + ->setType(DELETE_TYPE_REALTIME) + ->setTimestamp(time() - 60) + ->trigger(); } // # of days in seconds (1 day = 86400s) diff --git a/app/workers/builds.php b/app/workers/builds.php index 83197bdfa2..a665430965 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -13,43 +13,41 @@ use Utopia\Storage\Storage; use Utopia\Database\Document; use Utopia\Config\Config; -require_once __DIR__.'/../init.php'; +require_once __DIR__ . '/../init.php'; // Disable Auth since we already validate it in the API Authorization::disable(); Console::title('Builds V1 Worker'); -Console::success(APP_NAME.' build worker v1 has started'); +Console::success(APP_NAME . ' build worker v1 has started'); // TODO: Executor should return appropriate response codes. class BuildsV1 extends Worker { - /** - * @var Executor - */ - private $executor = null; + private ?Executor $executor = null; - public function getName(): string + public function getName(): string { return "builds"; } - public function init(): void { + public function init(): void + { $this->executor = new Executor(); } public function run(): void { $type = $this->args['type'] ?? ''; - $projectId = $this->args['projectId'] ?? ''; - $functionId = $this->args['resourceId'] ?? ''; - $deploymentId = $this->args['deploymentId'] ?? ''; + $project = new Document($this->args['project'] ?? []); + $resource = new Document($this->args['resource'] ?? []); + $deployment = new Document($this->args['deployment'] ?? []); switch ($type) { case BUILD_TYPE_DEPLOYMENT: case BUILD_TYPE_RETRY: - Console::info("Creating build for deployment: $deploymentId"); - $this->buildDeployment($projectId, $functionId, $deploymentId); + Console::info('Creating build for deployment: ' . $deployment->getId()); + $this->buildDeployment($project, $resource, $deployment); break; default: @@ -58,18 +56,16 @@ class BuildsV1 extends Worker } } - protected function buildDeployment(string $projectId, string $functionId, string $deploymentId) + protected function buildDeployment(Document $project, Document $function, Document $deployment) { - $dbForProject = $this->getProjectDB($projectId); - $dbForConsole = $this->getConsoleDB(); - $project = $dbForConsole->getDocument('projects', $projectId); + $dbForProject = $this->getProjectDB($project->getId()); - $function = $dbForProject->getDocument('functions', $functionId); + $function = $dbForProject->getDocument('functions', $function->getId()); if ($function->isEmpty()) { throw new Exception('Function not found', 404); } - $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); if ($deployment->isEmpty()) { throw new Exception('Deployment not found', 404); } @@ -91,7 +87,7 @@ class BuildsV1 extends Worker '$read' => [], '$write' => [], 'startTime' => $startTime, - 'deploymentId' => $deploymentId, + 'deploymentId' => $deployment->getId(), 'status' => 'processing', 'outputPath' => '', 'runtime' => $function->getAttribute('runtime'), @@ -103,7 +99,7 @@ class BuildsV1 extends Worker 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); } else { $build = $dbForProject->getDocument('builds', $buildId); } @@ -149,13 +145,13 @@ class BuildsV1 extends Worker try { $response = $this->executor->createRuntime( - projectId: $projectId, - deploymentId: $deploymentId, + projectId: $project->getId(), + deploymentId: $deployment->getId(), entrypoint: $deployment->getAttribute('entrypoint'), source: $source, - destination: APP_STORAGE_BUILDS . "/app-$projectId", - vars: $vars, - runtime: $key, + destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", + vars: $vars, + runtime: $key, baseImage: $baseImage, workdir: '/usr/code', remove: true, @@ -179,7 +175,7 @@ class BuildsV1 extends Worker /** Set auto deploy */ if ($deployment->getAttribute('activate') === true) { $function->setAttribute('deployment', $deployment->getId()); - $function = $dbForProject->updateDocument('functions', $functionId, $function); + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); } /** Update function schedule */ @@ -187,8 +183,7 @@ class BuildsV1 extends Worker $cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null; $next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0; $function->setAttribute('scheduleNext', (int)$next); - $function = $dbForProject->updateDocument('functions', $functionId, $function); - + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); } catch (\Throwable $th) { $endtime = \time(); $build->setAttribute('endTime', $endtime); @@ -213,5 +208,7 @@ class BuildsV1 extends Worker } } - public function shutdown(): void {} + public function shutdown(): void + { + } } diff --git a/src/Appwrite/Event/Build.php b/src/Appwrite/Event/Build.php new file mode 100644 index 0000000000..f6ba54dfd6 --- /dev/null +++ b/src/Appwrite/Event/Build.php @@ -0,0 +1,105 @@ +resource = $resource; + + return $this; + } + + /** + * Returns set resource document for the build event. + * + * @return null|\Utopia\Database\Document + */ + public function getResource(): ?Document + { + return $this->resource; + } + + /** + * Sets deployment for the build event. + * + * @param \Utopia\Database\Document $execution + * @return self + */ + public function setDeployment(Document $deployment): self + { + $this->deployment = $deployment; + + return $this; + } + + /** + * Returns set deployment for the build event. + * + * @return null|\Utopia\Database\Document + */ + public function getDeployment(): ?Document + { + return $this->deployment; + } + + /** + * Sets type for the build event. + * + * @param string $type Can be `BUILD_TYPE_DEPLOYMENT` or `BUILD_TYPE_RETRY`. + * @return self + */ + public function setType(string $type): self + { + $this->type = $type; + + return $this; + } + + /** + * Returns set type for the function event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + + /** + * Executes the function event and sends it to the functions worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + return Resque::enqueue($this->queue, $this->class, [ + 'project' => $this->project, + 'resource' => $this->resource, + 'deployment' => $this->deployment, + 'type' => $this->type + ]); + } +} diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 8b02cbd89f..742a759e09 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -8,6 +8,9 @@ use Utopia\Database\Document; class Delete extends Event { protected string $type = ''; + protected ?int $timestamp = null; + protected ?int $timestamp1d = null; + protected ?int $timestamp30m = null; protected ?Document $document = null; public function __construct() @@ -38,6 +41,27 @@ class Delete extends Event return $this->type; } + public function setTimestamp(int $timestamp): self + { + $this->timestamp = $timestamp; + + return $this; + } + + public function setTimestamp1d(int $timestamp): self + { + $this->timestamp1d = $timestamp; + + return $this; + } + + public function setTimestamp30m(int $timestamp): self + { + $this->timestamp30m = $timestamp; + + return $this; + } + /** * Sets the document for the delete event. * @@ -73,6 +97,9 @@ class Delete extends Event 'project' => $this->project, 'type' => $this->type, 'document' => $this->document, + 'timestamp' => $this->timestamp, + 'timestamp1d' => $this->timestamp1d, + 'timestamp30m' => $this->timestamp30m ]); } }