diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d4d5fc64cf..efd6967408 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -400,6 +400,7 @@ App::post('/v1/functions') $allEvents = Event::generateEvents('rules.[ruleId].create', [ 'ruleId' => $rule->getId(), ]); + $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], diff --git a/app/init.php b/app/init.php index 954e29410d..6de22d9526 100644 --- a/app/init.php +++ b/app/init.php @@ -202,6 +202,7 @@ const DELETE_TYPE_TOPIC = 'topic'; const DELETE_TYPE_TARGET = 'target'; const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets'; const DELETE_TYPE_SESSION_TARGETS = 'session_targets'; +const DELETE_TYPE_MAINTENANCE = 'maintenance'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; diff --git a/src/Appwrite/Event/Audit.php b/src/Appwrite/Event/Audit.php index 406f64b370..4b9aa9f5c5 100644 --- a/src/Appwrite/Event/Audit.php +++ b/src/Appwrite/Event/Audit.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Audit extends Event @@ -139,20 +138,13 @@ class Audit extends Event } /** - * Executes the event and sends it to the audit worker. + * Prepare payload for queue. * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'user' => $this->user, 'payload' => $this->payload, @@ -162,6 +154,6 @@ class Audit extends Event 'userAgent' => $this->userAgent, 'event' => $this->event, 'hostname' => $this->hostname - ]); + ]; } } diff --git a/src/Appwrite/Event/Build.php b/src/Appwrite/Event/Build.php index 1fbf20a9f9..831adf8e41 100644 --- a/src/Appwrite/Event/Build.php +++ b/src/Appwrite/Event/Build.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Build extends Event @@ -105,26 +104,19 @@ class Build extends Event } /** - * Executes the function event and sends it to the functions worker. + * Prepare payload for queue. * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'resource' => $this->resource, 'deployment' => $this->deployment, 'type' => $this->type, 'template' => $this->template - ]); + ]; } /** diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index 5d30c3d5ac..6a395417ed 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Certificate extends Event @@ -67,23 +66,16 @@ class Certificate extends Event } /** - * Executes the event and sends it to the certificates worker. + * Prepare the payload for the event * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'domain' => $this->domain, 'skipRenewCheck' => $this->skipRenewCheck - ]); + ]; } } diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index 1b0ea6851c..24123de6c1 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -4,7 +4,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; use Utopia\DSN\DSN; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Database extends Event @@ -100,18 +99,8 @@ class Database extends Event return $this->document; } - /** - * Executes the event and send it to the database worker. - * - * @return string|bool - * @throws \InvalidArgumentException - */ - public function trigger(): string|bool + public function getQueue(): string { - if ($this->paused) { - return false; - } - try { $dsn = new DSN($this->getProject()->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -119,23 +108,25 @@ class Database extends Event $dsn = new DSN('mysql://' . $this->getProject()->getAttribute('database')); } - $this->setQueue($dsn->getHost()); + $this->queue = $dsn->getHost(); + return $this->queue; + } - $client = new Client($this->queue, $this->connection); - - try { - $result = $client->enqueue([ - 'project' => $this->project, - 'user' => $this->user, - 'type' => $this->type, - 'collection' => $this->collection, - 'document' => $this->document, - 'database' => $this->database, - 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); - return $result; - } catch (\Throwable $th) { - return false; - } + /** + * Prepare the payload for the event + * + * @return array + */ + protected function preparePayload(): array + { + return [ + 'project' => $this->project, + 'user' => $this->user, + 'type' => $this->type, + 'collection' => $this->collection, + 'document' => $this->document, + 'database' => $this->database, + 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) + ]; } } diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 1a4c9318e3..f0af20f21b 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Delete extends Event @@ -131,22 +130,14 @@ class Delete extends Event return $this->document; } - /** - * Executes this event and sends it to the deletes worker. + * Prepare the payload for the event * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'type' => $this->type, 'document' => $this->document, @@ -154,6 +145,6 @@ class Delete extends Event 'resourceType' => $this->resourceType, 'datetime' => $this->datetime, 'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime - ]); + ]; } } diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index e3a2e394cf..5cd5f8e7d6 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -137,7 +137,6 @@ class Event public function setProject(Document $project): self { $this->project = $project; - return $this; } @@ -312,6 +311,27 @@ class Event return $this->params; } + /** + * Get trimmed values for sensitive/large payload fields. + * Override this method in child classes to add more fields to trim. + * + * @return array + */ + protected function trimPayload(): array + { + $trimmed = []; + + if ($this->project) { + $trimmed['project'] = new Document([ + '$id' => $this->project->getId(), + '$internalId' => $this->project->getInternalId(), + 'database' => $this->project->getAttribute('database') + ]); + } + + return $trimmed; + } + /** * Execute Event. * @@ -324,16 +344,30 @@ class Event return false; } - $client = new Client($this->queue, $this->connection); + /** The getter is required since events like Databases need to override the queue name depending on the project */ + $client = new Client($this->getQueue(), $this->connection); - return $client->enqueue([ + // Merge the base payload with any trimmed values + $payload = array_merge($this->preparePayload(), $this->trimPayload()); + + return $client->enqueue($payload); + } + + /** + * Prepare payload for queue. Can be overridden by child classes to customize payload. + * + * @return array + */ + protected function preparePayload(): array + { + return [ 'project' => $this->project, 'user' => $this->user, 'userId' => $this->userId, 'payload' => $this->payload, 'context' => $this->context, 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); + ]; } /** diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 0ad639a9f5..b3945fccb8 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Func extends Event @@ -173,13 +172,13 @@ class Func extends Event } /** - * Returns set custom data for the function event. + * Returns set JWT for the function event. * * @return string */ - public function getData(): string + public function getJWT(): string { - return $this->data; + return $this->jwt; } /** @@ -191,37 +190,19 @@ class Func extends Event public function setJWT(string $jwt): self { $this->jwt = $jwt; - return $this; } /** - * Returns set JWT for the function event. + * Prepare payload for the function event. * - * @return string + * @return array */ - public function getJWT(): string + protected function preparePayload(): array { - return $this->jwt; - } - - /** - * Executes the function event and sends it to the functions worker. - * - * @return string|bool - * @throws \InvalidArgumentException - */ - public function trigger(): string|bool - { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; - return $client->enqueue([ + return [ 'project' => $this->project, 'user' => $this->user, 'userId' => $this->userId, @@ -236,6 +217,6 @@ class Func extends Event 'path' => $this->path, 'headers' => $this->headers, 'method' => $this->method, - ]); + ]; } } diff --git a/src/Appwrite/Event/Mail.php b/src/Appwrite/Event/Mail.php index a0fca75688..1c9e539cdb 100644 --- a/src/Appwrite/Event/Mail.php +++ b/src/Appwrite/Event/Mail.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Mail extends Event @@ -397,20 +396,13 @@ class Mail extends Event } /** - * Executes the event and sends it to the mails worker. + * Prepare the payload for the event * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'recipient' => $this->recipient, 'name' => $this->name, @@ -421,6 +413,6 @@ class Mail extends Event 'variables' => $this->variables, 'attachment' => $this->attachment, 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); + ]; } } diff --git a/src/Appwrite/Event/Messaging.php b/src/Appwrite/Event/Messaging.php index 755b8c9158..61dbe9c427 100644 --- a/src/Appwrite/Event/Messaging.php +++ b/src/Appwrite/Event/Messaging.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Messaging extends Event @@ -176,19 +175,13 @@ class Messaging extends Event } /** - * Executes the event and sends it to the messaging worker. - * @return string|bool - * @throws \InvalidArgumentException + * Prepare the payload for the event + * + * @return array */ - public function trigger(): string | bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'type' => $this->type, 'project' => $this->project, 'user' => $this->user, @@ -196,6 +189,6 @@ class Messaging extends Event 'message' => $this->message, 'recipients' => $this->recipients, 'providerType' => $this->providerType, - ]); + ]; } } diff --git a/src/Appwrite/Event/Migration.php b/src/Appwrite/Event/Migration.php index 789b8e2160..5fb2d5a106 100644 --- a/src/Appwrite/Event/Migration.php +++ b/src/Appwrite/Event/Migration.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Migration extends Event @@ -68,23 +67,16 @@ class Migration extends Event } /** - * Executes the migration event and sends it to the migrations worker. + * Prepare the payload for the migration event. * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'user' => $this->user, 'migration' => $this->migration, - ]); + ]; } } diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 89e900d2ab..5609859f37 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Usage extends Event @@ -51,6 +50,20 @@ class Usage extends Event return $this; } + /** + * Prepare the payload for the usage event. + * + * @return array + */ + protected function preparePayload(): array + { + return [ + 'project' => $this->project, + 'reduce' => $this->reduce, + 'metrics' => $this->metrics, + ]; + } + /** * Sends metrics to the usage worker. * @@ -58,20 +71,8 @@ class Usage extends Event */ public function trigger(): string|bool { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - $result = $client->enqueue([ - 'project' => $this->getProject(), - 'reduce' => $this->reduce, - 'metrics' => $this->metrics, - ]); - + parent::trigger(); $this->metrics = []; - - return $result; + return true; } } diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php index 2998e4e104..6f44de4eda 100644 --- a/src/Appwrite/Event/UsageDump.php +++ b/src/Appwrite/Event/UsageDump.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class UsageDump extends Event @@ -32,20 +31,14 @@ class UsageDump extends Event } /** - * Sends metrics to the usage worker. + * Prepare the payload for the usage dump event. * - * @return string|bool + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'stats' => $this->stats, - ]); + ]; } } diff --git a/src/Appwrite/Event/Webhook.php b/src/Appwrite/Event/Webhook.php index 36c6923cae..3e0dbe446f 100644 --- a/src/Appwrite/Event/Webhook.php +++ b/src/Appwrite/Event/Webhook.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Database\Document; use Utopia\Queue\Connection; class Webhook extends Event @@ -16,15 +15,17 @@ class Webhook extends Event ->setClass(Event::WEBHOOK_CLASS_NAME); } - public function trigger(): string|bool + /** + * Trim the payload for the webhook event. + * + * @return array + */ + public function trimPayload(): array { - /** Filter out context and trim project to keep the payload small */ - $this->context = []; - $this->project = new Document([ - '$id' => $this->project->getId(), - '$internalId' => $this->project->getInternalId(), - ]); - - return parent::trigger(); + $trimmed = parent::trimPayload(); + if (isset($this->context)) { + $trimmed['context'] = []; + } + return $trimmed; } } diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index c789cbdaac..a7558934b5 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -64,11 +64,9 @@ class Maintenance extends Action */ protected function notifyProjects(Delete $queueForDeletes, int $usageStatsRetentionHourly): void { - $this->notifyDeleteTargets($queueForDeletes); - $this->notifyDeleteExecutionLogs($queueForDeletes); - $this->notifyDeleteAuditLogs($queueForDeletes); - $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes); - $this->notifyDeleteExpiredSessions($queueForDeletes); + $queueForDeletes + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) + ->trigger(); } protected function foreachProject(Database $dbForPlatform, callable $callback): void @@ -98,28 +96,6 @@ class Maintenance extends Action Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); } - private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_EXECUTIONS) - ->trigger(); - } - - private function notifyDeleteAuditLogs(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_AUDIT) - ->trigger(); - } - - private function notifyDeleteUsageStats(int $usageStatsRetentionHourly, Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_USAGE) - ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) - ->trigger(); - } - private function notifyDeleteConnections(Delete $queueForDeletes): void { $queueForDeletes @@ -128,13 +104,6 @@ class Maintenance extends Action ->trigger(); } - private function notifyDeleteExpiredSessions(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_SESSIONS) - ->trigger(); - } - private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void { $time = DateTime::now(); @@ -177,11 +146,4 @@ class Maintenance extends Action ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) ->trigger(); } - - private function notifyDeleteTargets(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_EXPIRED_TARGETS) - ->trigger(); - } } diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index bef78a7514..4f5d6eb694 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -46,6 +46,7 @@ class Builds extends Action $this ->desc('Builds worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('queueForEvents') ->inject('queueForFunctions') @@ -54,11 +55,12 @@ class Builds extends Action ->inject('dbForProject') ->inject('deviceForFunctions') ->inject('log') - ->callback(fn ($message, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); } /** * @param Message $message + * @param Document $project * @param Database $dbForPlatform * @param Event $queueForEvents * @param Func $queueForFunctions @@ -70,7 +72,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -79,7 +81,6 @@ class Builds extends Action } $type = $payload['type'] ?? ''; - $project = new Document($payload['project'] ?? []); $resource = new Document($payload['resource'] ?? []); $deployment = new Document($payload['deployment'] ?? []); $template = new Document($payload['template'] ?? []); diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 5b73e6a75c..9345f31165 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -34,21 +34,23 @@ class Databases extends Action $this ->desc('Databases worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('dbForProject') ->inject('log') - ->callback(fn (Message $message, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $dbForPlatform, $dbForProject, $log)); + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log)); } /** * @param Message $message + * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject * @param Log $log * @return void * @throws \Exception */ - public function action(Message $message, Database $dbForPlatform, Database $dbForProject, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log): void { $payload = $message->getPayload() ?? []; @@ -57,7 +59,6 @@ class Databases extends Action } $type = $payload['type']; - $project = new Document($payload['project']); $collection = new Document($payload['collection'] ?? []); $document = new Document($payload['document'] ?? []); $database = new Document($payload['database'] ?? []); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9aaf19f412..539bbd61f9 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -43,6 +43,7 @@ class Deletes extends Action $this ->desc('Deletes worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('timelimit') @@ -55,8 +56,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $project, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -64,7 +65,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -78,7 +79,6 @@ class Deletes extends Action $resource = $payload['resource'] ?? null; $resourceType = $payload['resourceType'] ?? null; $document = new Document($payload['document'] ?? []); - $project = new Document($payload['project'] ?? []); $log->addTag('projectId', $project->getId()); $log->addTag('type', $type); @@ -153,6 +153,13 @@ class Deletes extends Action case DELETE_TYPE_SESSION_TARGETS: $this->deleteSessionTargets($project, $getProjectDB, $document); break; + case DELETE_TYPE_MAINTENANCE: + $this->deleteExpiredTargets($project, $getProjectDB); + $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); + $this->deleteAuditLogs($project, $getProjectDB, $auditRetention); + $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); + $this->deleteExpiredSessions($project, $getProjectDB); + break; default: throw new \Exception('No delete operation for type: ' . \strval($type)); } diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 6e27f16c2a..7837957c5d 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -59,15 +59,17 @@ class Messaging extends Action $this ->desc('Messaging worker') ->inject('message') + ->inject('project') ->inject('log') ->inject('dbForProject') ->inject('deviceForFiles') ->inject('queueForUsage') - ->callback(fn (Message $message, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $log, $dbForProject, $deviceForFiles, $queueForUsage)); + ->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForUsage)); } /** * @param Message $message + * @param Document $project * @param Log $log * @param Database $dbForProject * @param Device $deviceForFiles @@ -77,6 +79,7 @@ class Messaging extends Action */ public function action( Message $message, + Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, @@ -90,7 +93,6 @@ class Messaging extends Action } $type = $payload['type'] ?? ''; - $project = new Document($payload['project'] ?? []); switch ($type) { case MESSAGE_SEND_TYPE_INTERNAL: diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f6af0eb5f2..078c9fa0ff 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -51,16 +51,17 @@ class Migrations extends Action $this ->desc('Migrations worker') ->inject('message') + ->inject('project') ->inject('dbForProject') ->inject('dbForPlatform') ->inject('logError') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $dbForProject, $dbForPlatform, $logError)); + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForPlatform, callable $logError): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError): void { $payload = $message->getPayload() ?? []; @@ -69,7 +70,6 @@ class Migrations extends Action } $events = $payload['events'] ?? []; - $project = new Document($payload['project'] ?? []); $migration = new Document($payload['migration'] ?? []); if ($project->getId() === 'console') { diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 8199fe73a7..3687eeab67 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -34,10 +34,11 @@ class Usage extends Action $this ->desc('Usage worker') ->inject('message') + ->inject('project') ->inject('getProjectDB') ->inject('queueForUsageDump') - ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { - $this->action($message, $getProjectDB, $queueForUsageDump); + ->callback(function (Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $project, $getProjectDB, $queueForUsageDump); }); $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); @@ -46,21 +47,20 @@ class Usage extends Action /** * @param Message $message + * @param Document $project * @param callable $getProjectDB * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void + public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } - $document = $payload['project'] ?? []; - $project = new Document($document); if (empty($project->getAttribute('database'))) { var_dump($payload); diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 3e50ba0363..2f1d13f29a 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -59,26 +59,11 @@ class UsageDump extends Action foreach ($payload['stats'] ?? [] as $stats) { - //$project = new Document($stats['project'] ?? []); - - /** - * Start temp bug fallback - */ - $document = $stats['project'] ?? []; - if (!empty($document['$uid'])) { - $document['$id'] = $document['$uid']; - } - - $project = new Document($document); - - if (empty($project->getAttribute('database'))) { - continue; - } + $project = new Document($stats['project'] ?? []); /** * End temp bug fallback */ - $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; $receivedAt = $stats['receivedAt'] ?? 'NONE'; if ($numberOfKeys === 0) { diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 50f64ea02d..a76e4f17b0 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -32,22 +32,24 @@ class Webhooks extends Action $this ->desc('Webhooks worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('queueForMails') ->inject('queueForUsage') ->inject('log') - ->callback(fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $dbForPlatform, $queueForMails, $queueForUsage, $log)); + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForUsage, $log)); } /** * @param Message $message + * @param Document $project * @param Database $dbForPlatform * @param Mail $queueForMails * @param Log $log * @return void * @throws Exception */ - public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void { $this->errors = []; $payload = $message->getPayload() ?? []; @@ -60,8 +62,6 @@ class Webhooks extends Action $webhookPayload = json_encode($payload['payload']); $user = new Document($payload['user'] ?? []); - $project = new Document($payload['project']); - $project = $dbForPlatform->getDocument('projects', $project->getId()); $log->addTag('projectId', $project->getId()); foreach ($project->getAttribute('webhooks', []) as $webhook) { diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 9b9f03a100..d8d1eb8eb5 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -375,7 +375,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(200, $deployment['headers']['status-code']); $this->assertEquals('ready', $deployment['body']['status']); - }, 500000, 1000); + }, 50000, 1000); $function = $this->getFunction($functionId);