From 375f6432142f462fe8a052c1ea7138dea4755fed Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 03:29:32 +0000 Subject: [PATCH 01/65] refactor: triggering realtime events with queueForRealtime --- app/controllers/api/functions.php | 52 ++++++--------- app/worker.php | 5 ++ src/Appwrite/Event/Event.php | 23 +++++++ src/Appwrite/Event/Realtime.php | 2 +- src/Appwrite/Platform/Workers/Databases.php | 74 +++++++++------------ 5 files changed, 80 insertions(+), 76 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 14255ef7a4..0f9f5211f4 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -6,13 +6,13 @@ use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\Validator\FunctionEvent; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\Validator\Headers; use Appwrite\Functions\Validator\RuntimeSpecification; -use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -194,9 +194,10 @@ App::post('/v1/functions') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') + ->inject('queueForRealtime') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Realtime $queueForRealtime, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check @@ -386,18 +387,18 @@ App::post('/v1/functions') ])) ); - /** Trigger Webhook */ $ruleModel = new Rule(); $ruleCreate = $queueForEvents - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setQueue(Event::WEBHOOK_QUEUE_NAME); + ->setProject($project) + ->setEvent('rules.[ruleId].create') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + /** Trigger Webhook */ $ruleCreate - ->setProject($project) - ->setEvent('rules.[ruleId].create') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->trigger(); /** Trigger Functions */ @@ -406,31 +407,16 @@ App::post('/v1/functions') ->setQueue(Event::FUNCTIONS_QUEUE_NAME) ->trigger(); - /** Trigger realtime event */ - $allEvents = Event::generateEvents('rules.[ruleId].create', [ - 'ruleId' => $rule->getId(), - ]); + /** Trigger Realtime Events */ + $queueForRealtime + ->from($ruleCreate) + ->setProjectId('console') + ->trigger(); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $rule, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->from($ruleCreate) + ->setProjectId($project->getId()) + ->trigger(); } $queueForEvents->setParam('functionId', $function->getId()); diff --git a/app/worker.php b/app/worker.php index 605474e9f1..ad6bf475f9 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,6 +13,7 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; /** remove */ @@ -317,6 +318,10 @@ Server::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); +Server::setResource('queueForRealtime', function () { + return new Realtime(); +}, []); + Server::setResource('queueForCertificates', function (Publisher $publisher) { return new Certificate($publisher); }, ['publisher']); diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 0edffdf4dc..8085a836a8 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -66,6 +66,7 @@ class Event protected ?Document $project = null; protected ?Document $user = null; protected ?string $userId = null; + protected ?string $projectId = null; protected bool $paused = false; /** @@ -151,6 +152,18 @@ class Event return $this; } + /** + * Set projectId for this event. + * + * @param string $projectId + * @return self + */ + public function setProjectId(string $projectId): self + { + $this->projectId = $projectId; + return $this; + } + /** * Get project for this event. * @@ -161,6 +174,16 @@ class Event return $this->project; } + /** + * Get projectId for this event. + * + * @return ?string + */ + public function getProjectId(): ?string + { + return $this->projectId; + } + /** * Set user for this event. * diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index f4f00b59d4..8c302bbabf 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -54,7 +54,7 @@ class Realtime extends Event ); RealtimeAdapter::send( - projectId: $target['projectId'] ?? $this->getProject()->getId(), + projectId: $this->getProjectId() ?? $target['projectId'] ?? $this->getProject()->getId(), payload: $this->getRealtimePayload(), events: $allEvents, channels: $target['channels'], diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 441b09b4cc..50a3fa52f3 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Event; -use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Event\Realtime; use Exception; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -38,7 +38,8 @@ class Databases extends Action ->inject('dbForPlatform') ->inject('dbForProject') ->inject('log') - ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log)); + ->inject('queueForRealtime') + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log, $queueForRealtime)); } /** @@ -47,10 +48,11 @@ class Databases extends Action * @param Database $dbForPlatform * @param Database $dbForProject * @param Log $log + * @param Realtime $queueForRealtime * @return void * @throws \Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime): void { $payload = $message->getPayload() ?? []; @@ -75,10 +77,10 @@ class Databases extends Action match (\strval($type)) { DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), - DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject), - DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject), - DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject), - DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject), + DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), + DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), + DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), + DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), default => throw new \Exception('No database operation for type: ' . \strval($type)), }; } @@ -90,13 +92,14 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict * @throws \Exception * @throws \Throwable */ - private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -106,12 +109,6 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'attributeId' => $attribute->getId() - ]); /** * TODO @christyjacob4 verify if this is still the case * Fetch attribute from the database, since with Resque float values are loosing informations. @@ -200,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $projectId, $events); + $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -217,13 +214,14 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict * @throws \Exception * @throws \Throwable **/ - private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -312,7 +310,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $projectId, $events); + $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -358,7 +356,7 @@ class Databases extends Action } if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject); + $this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject, $queueForRealtime); } else { $dbForProject->updateDocument('indexes', $index->getId(), $index); } @@ -381,6 +379,7 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict @@ -388,7 +387,7 @@ class Databases extends Action * @throws DatabaseException * @throws \Throwable */ - private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -430,7 +429,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $projectId, $events); + $this->trigger($database, $collection, $index, $project, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -442,6 +441,7 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict @@ -449,7 +449,7 @@ class Databases extends Action * @throws DatabaseException * @throws \Throwable */ - private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -490,7 +490,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $projectId, $events); + $this->trigger($database, $collection, $index, $project, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -617,26 +617,16 @@ class Databases extends Action Document $collection, Document $attribute, Document $project, - string $projectId, - array $events + Realtime $queueForRealtime ): void { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project, - ); - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') + ->setParam('databaseId', $database->getId()) + ->setParam('collectionId', $collection->getId()) + ->setParam('attributeId', $attribute->getId()) + ->setPayload($attribute->getArrayCopy()) + ->trigger(); } } From 2c4c42de054ad6388f53562c29621b913a58864d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 03:53:18 +0000 Subject: [PATCH 02/65] docs: fix database worker --- src/Appwrite/Platform/Workers/Databases.php | 40 ++++++++------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 50a3fa52f3..6db88a618f 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\Event; use Appwrite\Event\Realtime; use Exception; use Utopia\CLI\Console; @@ -57,7 +56,7 @@ class Databases extends Action $payload = $message->getPayload() ?? []; if (empty($payload)) { - throw new \Exception('Missing payload'); + throw new Exception('Missing payload'); } $type = $payload['type']; @@ -81,7 +80,7 @@ class Databases extends Action DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), - default => throw new \Exception('No database operation for type: ' . \strval($type)), + default => throw new Exception('No database operation for type: ' . \strval($type)), }; } @@ -166,7 +165,7 @@ class Databases extends Action break; default: if (!$dbForProject->createAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) { - throw new \Exception('Failed to create Attribute'); + throw new Exception('Failed to create Attribute'); } } @@ -231,15 +230,8 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'attributeId' => $attribute->getId() - ]); $collectionId = $collection->getId(); $key = $attribute->getAttribute('key', ''); - $status = $attribute->getAttribute('status', ''); $type = $attribute->getAttribute('type', ''); $project = $dbForPlatform->getDocument('projects', $projectId); $options = $attribute->getAttribute('options', []); @@ -397,12 +389,6 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'indexId' => $index->getId() - ]); $collectionId = $collection->getId(); $key = $index->getAttribute('key', ''); $type = $index->getAttribute('type', ''); @@ -459,12 +445,6 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'indexId' => $index->getId() - ]); $key = $index->getAttribute('key'); $status = $index->getAttribute('status', ''); $project = $dbForPlatform->getDocument('projects', $projectId); @@ -568,14 +548,14 @@ class Databases extends Action /** - * @param string $collection collectionID + * @param string $collectionId * @param array $queries * @param Database $database * @param callable|null $callback * @return void * @throws Exception */ - protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + protected function deleteByGroup(string $collectionId, array $queries, Database $database, callable $callback = null): void { $count = 0; $chunk = 0; @@ -587,7 +567,7 @@ class Databases extends Action while ($sum === $limit) { $chunk++; - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + $results = $database->find($collectionId, \array_merge([Query::limit($limit)], $queries)); $sum = count($results); @@ -612,6 +592,14 @@ class Databases extends Action Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } + /** + * @param Document $database + * @param Document $collection + * @param Document $attribute + * @param Document $project + * @param Realtime $queueForRealtime + * @return void + */ protected function trigger( Document $database, Document $collection, From 47fbb777ed2893e8230316d0386febed76e407a4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 04:03:02 +0000 Subject: [PATCH 03/65] chore: refactor migrations realtime queue --- src/Appwrite/Platform/Workers/Migrations.php | 66 +++++++++----------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index cd567f6fa3..50d6002d28 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -3,8 +3,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; -use Appwrite\Event\Event; -use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Event\Realtime; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -54,13 +53,14 @@ class Migrations extends Action ->inject('dbForProject') ->inject('dbForPlatform') ->inject('logError') - ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError)); + ->inject('queueForRealtime') + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime): void { $payload = $message->getPayload() ?? []; @@ -87,7 +87,7 @@ class Migrations extends Action return; } - $this->processMigration($migration); + $this->processMigration($migration, $queueForRealtime); } /** @@ -155,34 +155,24 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function updateMigrationDocument(Document $migration, Document $project): Document + protected function updateMigrationDocument(Document $migration, Document $project, Realtime $queueForRealtime): Document { - /** Trigger Realtime */ - $allEvents = Event::generateEvents('migrations.[migrationId].update', [ - 'migrationId' => $migration->getId(), - ]); + /** Trigger Realtime Events */ + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent('migrations.[migrationId].update') + ->setParam('migrationId', $migration->getId()) + ->setPayload($migration->getArrayCopy()) + ->trigger(); - $target = Realtime::fromPayload( - event: $allEvents[0], - payload: $migration, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $migration->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - ); - - Realtime::send( - projectId: $project->getId(), - payload: $migration->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - ); + $queueForRealtime + ->setProject($project) + ->setProjectId($project->getId()) + ->setEvent('migrations.[migrationId].update') + ->setParam('migrationId', $migration->getId()) + ->setPayload($migration->getArrayCopy()) + ->trigger(); return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); } @@ -241,7 +231,7 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration): void + protected function processMigration(Document $migration, Realtime $queueForRealtime): void { $project = $this->project; $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); @@ -265,7 +255,7 @@ class Migrations extends Action $migration->setAttribute('stage', 'processing'); $migration->setAttribute('status', 'processing'); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); $source = $this->processSource($migration); $destination = $this->processDestination($migration, $tempAPIKey); @@ -279,14 +269,14 @@ class Migrations extends Action /** Start Transfer */ $migration->setAttribute('stage', 'migrating'); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); $transfer->run( $migration->getAttribute('resources'), - function () use ($migration, $transfer, $projectDocument) { + function () use ($migration, $transfer, $projectDocument, $queueForRealtime) { $migration->setAttribute('resourceData', json_encode($transfer->getCache())); $migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); }, $migration->getAttribute('resourceId'), $migration->getAttribute('resourceType') @@ -323,7 +313,7 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); return; } @@ -364,7 +354,7 @@ class Migrations extends Action $migration->setAttribute('errors', $errorMessages); } } finally { - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); if ($migration->getAttribute('status', '') === 'failed') { Console::error('Migration('.$migration->getInternalId().':'.$migration->getId().') failed, Project('.$this->project->getInternalId().':'.$this->project->getId().')'); From bcdae7f4e37ff7d348d54f39f2991676a89e77c4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 04:21:36 +0000 Subject: [PATCH 04/65] chore: refactor certificate realtime, fix database --- .../Platform/Workers/Certificates.php | 76 ++++++++----------- src/Appwrite/Platform/Workers/Databases.php | 23 +++--- 2 files changed, 46 insertions(+), 53 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 7e220b2734..c7f345f77a 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -6,7 +6,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; -use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Event\Realtime; use Appwrite\Network\Validator\CNAME; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; @@ -47,11 +47,12 @@ class Certificates extends Action ->inject('queueForMails') ->inject('queueForEvents') ->inject('queueForFunctions') + ->inject('queueForRealtime') ->inject('log') ->inject('certificates') ->callback( - fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) => - $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates) + fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates) ); } @@ -61,13 +62,14 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param Log $log * @param CertificatesAdapter $certificates * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void + public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -81,7 +83,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck); + $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck); } /** @@ -90,13 +92,14 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param CertificatesAdapter $certificates * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -186,7 +189,7 @@ class Certificates extends Action $certificate->setAttribute('updated', DateTime::now()); // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions); + $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); } } @@ -199,13 +202,14 @@ class Certificates extends Action * @param Database $dbForPlatform Database connection for console * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @return void * @throws \Utopia\Database\Exception * @throws Authorization * @throws Conflict * @throws Structure */ - private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void + private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void { // Check if update or insert required $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); @@ -219,7 +223,7 @@ class Certificates extends Action } $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions); + $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); } /** @@ -338,7 +342,7 @@ class Certificates extends Action * * @return void */ - private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void + private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void { // TODO: @christyjacob remove once we migrate the rules in 1.7.x if (System::getEnv('_APP_RULES_FORMAT') === 'md5') { @@ -367,50 +371,34 @@ class Certificates extends Action return; } - /** Trigger Webhook */ $ruleModel = new Rule(); + $queueForEvents + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + + /** Trigger Webhook */ $queueForEvents ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) ->trigger(); - /** Trigger Functions */ $queueForFunctions - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) + ->from($queueForEvents) ->trigger(); - /** Trigger realtime event */ - $allEvents = Event::generateEvents('rules.[ruleId].update', [ - 'ruleId' => $rule->getId(), - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $rule, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + /** Trigger Realtime Events */ + $queueForRealtime + ->from($queueForEvents) + ->setProjectId('console') + ->trigger(); + + $queueForRealtime + ->from($queueForEvents) + ->setProjectId($project->getId()) + ->trigger(); } } } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 6db88a618f..b617da3e33 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -36,9 +36,9 @@ class Databases extends Action ->inject('project') ->inject('dbForPlatform') ->inject('dbForProject') - ->inject('log') ->inject('queueForRealtime') - ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log, $queueForRealtime)); + ->inject('log') + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $queueForRealtime, $log)); } /** @@ -46,12 +46,12 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject - * @param Log $log * @param Realtime $queueForRealtime + * @param Log $log * @return void * @throws \Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime): void + public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime, Log $log): void { $payload = $message->getPayload() ?? []; @@ -108,6 +108,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = "databases.[databaseId].collections.[collectionId].attributes.[attributeId].update"; /** * TODO @christyjacob4 verify if this is still the case * Fetch attribute from the database, since with Resque float values are loosing informations. @@ -196,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); + $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -230,6 +231,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete'; $collectionId = $collection->getId(); $key = $attribute->getAttribute('key', ''); $type = $attribute->getAttribute('type', ''); @@ -302,7 +304,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); + $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -389,6 +391,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update'; $collectionId = $collection->getId(); $key = $index->getAttribute('key', ''); $type = $index->getAttribute('type', ''); @@ -415,7 +418,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $queueForRealtime); + $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -445,6 +448,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete'; $key = $index->getAttribute('key'); $status = $index->getAttribute('status', ''); $project = $dbForPlatform->getDocument('projects', $projectId); @@ -470,7 +474,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $queueForRealtime); + $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -605,12 +609,13 @@ class Databases extends Action Document $collection, Document $attribute, Document $project, + string $event, Realtime $queueForRealtime ): void { $queueForRealtime ->setProject($project) ->setProjectId('console') - ->setEvent('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') + ->setEvent($event) ->setParam('databaseId', $database->getId()) ->setParam('collectionId', $collection->getId()) ->setParam('attributeId', $attribute->getId()) From f666e18154ecf2b9b0e51dd3c64b2fe400cd0c80 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 05:00:45 +0000 Subject: [PATCH 05/65] chore: fix databases worker realtime queueing --- src/Appwrite/Platform/Workers/Databases.php | 30 ++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index b617da3e33..b0c6a9df73 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -197,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -304,7 +304,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -418,7 +418,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -474,7 +474,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -599,16 +599,20 @@ class Databases extends Action /** * @param Document $database * @param Document $collection - * @param Document $attribute * @param Document $project + * @param Document|null $attribute + * @param Document|null $index + * @param Document $payload * @param Realtime $queueForRealtime * @return void */ protected function trigger( Document $database, Document $collection, - Document $attribute, Document $project, + Document|null $attribute = null, + Document|null $index = null, + Document $payload, string $event, Realtime $queueForRealtime ): void { @@ -617,9 +621,17 @@ class Databases extends Action ->setProjectId('console') ->setEvent($event) ->setParam('databaseId', $database->getId()) - ->setParam('collectionId', $collection->getId()) - ->setParam('attributeId', $attribute->getId()) - ->setPayload($attribute->getArrayCopy()) + ->setParam('collectionId', $collection->getId()); + + if ($attribute !== null) { + $queueForRealtime->setParam('attributeId', $attribute->getId()); + } + if ($index !== null) { + $queueForRealtime->setParam('indexId', $index->getId()); + } + + $queueForRealtime + ->setPayload($payload->getArrayCopy()) ->trigger(); } } From 532160705ffbde2ec6f008546d66ddcbe5e8048c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 05:50:30 +0000 Subject: [PATCH 06/65] chore: refactor realtime queueing in functions worker --- src/Appwrite/Platform/Workers/Functions.php | 58 +++++++++------------ 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 0a7c39c02f..f485db8648 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -5,8 +5,8 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; -use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Execution; use Exception; use Executor\Executor; @@ -45,14 +45,15 @@ class Functions extends Action ->inject('message') ->inject('dbForProject') ->inject('queueForFunctions') + ->inject('queueForRealtime') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('log') ->inject('isResourceBlocked') - ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); + ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); } - public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void + public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void { $payload = $message->getPayload() ?? []; @@ -137,6 +138,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, + queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, @@ -177,6 +179,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, + queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, @@ -199,6 +202,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, + queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, @@ -284,6 +288,7 @@ class Functions extends Action * @param Log $log * @param Database $dbForProject * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param StatsUsage $queueForStatsUsage * @param Event $queueForEvents * @param Document $project @@ -308,6 +313,7 @@ class Functions extends Action Log $log, Database $dbForProject, Func $queueForFunctions, + Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Event $queueForEvents, Document $project, @@ -564,20 +570,21 @@ class Functions extends Action ; } - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - /** Trigger Webhook */ $executionModel = new Execution(); $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) ->setProject($project) ->setUser($user) ->setEvent('functions.[functionId].executions.[executionId].update') ->setParam('functionId', $function->getId()) ->setParam('executionId', $execution->getId()) - ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) + ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))); + + /** Trigger Webhook */ + $queueForEvents + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ->setClass(Event::WEBHOOK_CLASS_NAME) ->trigger(); /** Trigger Functions */ @@ -585,31 +592,16 @@ class Functions extends Action ->from($queueForEvents) ->trigger(); - /** Trigger realtime event */ - $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ - 'functionId' => $function->getId(), - 'executionId' => $execution->getId() - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $execution, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + /** Trigger Realtime Events */ + $queueForRealtime + ->from($queueForEvents) + ->setProjectId('console') + ->trigger(); + + $queueForRealtime + ->from($queueForEvents) + ->setProjectId($project->getId()) + ->trigger(); if (!empty($error)) { throw new Exception($error, $errorCode); From 5076303c33a600da6b91a4a8ebe7cb18d79d325c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 06:06:43 +0000 Subject: [PATCH 07/65] chore: refactor realtime queueing in builds worker --- src/Appwrite/Platform/Workers/Builds.php | 115 +++++++++-------------- 1 file changed, 47 insertions(+), 68 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index e7cbbd5088..f3f733f5eb 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -5,8 +5,8 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; -use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -50,12 +50,13 @@ class Builds extends Action ->inject('dbForPlatform') ->inject('queueForEvents') ->inject('queueForFunctions') + ->inject('queueForRealtime') ->inject('queueForStatsUsage') ->inject('cache') ->inject('dbForProject') ->inject('deviceForFunctions') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); } /** @@ -64,6 +65,7 @@ class Builds extends Action * @param Database $dbForPlatform * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param StatsUsage $queueForStatsUsage * @param Cache $cache * @param Database $dbForProject @@ -72,7 +74,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -93,7 +95,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); + $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); break; default: @@ -104,6 +106,7 @@ class Builds extends Action /** * @param Device $deviceForFunctions * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param Event $queueForEvents * @param StatsUsage $queueForStatsUsage * @param Database $dbForPlatform @@ -118,7 +121,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -152,10 +155,7 @@ class Builds extends Action } // Realtime preparation - $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ - 'functionId' => $function->getId(), - 'deploymentId' => $deployment->getId() - ]); + $event = "functions.[functionId].deployments.[deploymentId].update"; $startTime = DateTime::now(); $durationStart = \microtime(true); @@ -369,21 +369,16 @@ class Builds extends Action $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); /** - * Send realtime Event + * Trigger Realtime Event */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); } $tmpPath = '/tmp/builds/' . $buildId; @@ -449,21 +444,15 @@ class Builds extends Action ->from($deploymentUpdate) ->trigger(); - /** Trigger Realtime */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + /** Trigger Realtime Event */ + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); $vars = []; @@ -556,12 +545,12 @@ class Builds extends Action $err = $error; } }), - Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) { + Co\go(function () use ($executor, $project, $function, $deployment, &$response, &$build, $dbForProject, $event, &$err, $queueForRealtime, &$isCanceled) { try { $executor->getLogs( deploymentId: $deployment->getId(), projectId: $project->getId(), - callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) { + callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $event, $project, $function, $deployment, $queueForRealtime, &$isCanceled) { if ($isCanceled) { return; } @@ -586,21 +575,16 @@ class Builds extends Action $build = $dbForProject->updateDocument('builds', $build->getId(), $build); /** - * Send realtime Event + * Trigger Realtime Event */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); } } ); @@ -688,21 +672,16 @@ class Builds extends Action } } finally { /** - * Send realtime Event + * Trigger Realtime Event */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); /** Trigger usage queue */ if ($build->getAttribute('status') === 'ready') { From 940f4d12af617977b5f8312658e0454dee4f7d5f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 11 Feb 2025 07:52:37 +0000 Subject: [PATCH 08/65] Remove usage and usage dump in favor of stats-usage and stats-usage-dump --- Dockerfile | 4 +- app/init.php | 3 - app/worker.php | 8 - bin/worker-usage | 3 - bin/worker-usage-dump | 3 - src/Appwrite/Platform/Services/Workers.php | 8 - src/Appwrite/Platform/Workers/Usage.php | 292 -------------------- src/Appwrite/Platform/Workers/UsageDump.php | 286 ------------------- 8 files changed, 1 insertion(+), 606 deletions(-) delete mode 100644 bin/worker-usage delete mode 100644 bin/worker-usage-dump delete mode 100644 src/Appwrite/Platform/Workers/Usage.php delete mode 100644 src/Appwrite/Platform/Workers/UsageDump.php diff --git a/Dockerfile b/Dockerfile index 2bb9f80d9e..01ad5d3fba 100755 --- a/Dockerfile +++ b/Dockerfile @@ -88,9 +88,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-stats-usage && \ chmod +x /usr/local/bin/worker-stats-usage-dump && \ chmod +x /usr/local/bin/stats-resources && \ - chmod +x /usr/local/bin/worker-stats-resources && \ - chmod +x /usr/local/bin/worker-usage && \ - chmod +x /usr/local/bin/worker-usage-dump + chmod +x /usr/local/bin/worker-stats-resources # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/init.php b/app/init.php index 0c6746de3c..9bd8cf839c 100644 --- a/app/init.php +++ b/app/init.php @@ -1197,9 +1197,6 @@ App::setResource('queueForAudits', function (Queue\Publisher $publisher) { App::setResource('queueForFunctions', function (Queue\Publisher $publisher) { return new Func($publisher); }, ['publisher']); -App::setResource('queueForUsage', function (Queue\Publisher $publisher) { - return new Usage($publisher); -}, ['publisher']); App::setResource('queueForCertificates', function (Queue\Publisher $publisher) { return new Certificate($publisher); }, ['publisher']); diff --git a/app/worker.php b/app/worker.php index 605474e9f1..dc02da1bb7 100644 --- a/app/worker.php +++ b/app/worker.php @@ -269,14 +269,6 @@ Server::setResource('consumer', function (Group $pools) { return $pools->get('consumer')->pop()->getResource(); }, ['pools']); -Server::setResource('queueForUsage', function (Publisher $publisher) { - return new Usage($publisher); -}, ['publisher']); - -Server::setResource('queueForUsageDump', function (Publisher $publisher) { - return new UsageDump($publisher); -}, ['publisher']); - Server::setResource('queueForStatsUsage', function (Publisher $publisher) { return new StatsUsage($publisher); }, ['publisher']); diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index e39ce8477c..0000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php usage $@ \ No newline at end of file diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump deleted file mode 100644 index 43ca87fcb3..0000000000 --- a/bin/worker-usage-dump +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index e121ee35f7..4f4095aca4 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -14,10 +14,6 @@ use Appwrite\Platform\Workers\Migrations; use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\StatsUsage; use Appwrite\Platform\Workers\StatsUsageDump; -/** remove */ -use Appwrite\Platform\Workers\Usage; -use Appwrite\Platform\Workers\UsageDump; -/** /remove */ use Appwrite\Platform\Workers\Webhooks; use Utopia\Platform\Service; @@ -40,10 +36,6 @@ class Workers extends Service ->addAction(StatsUsage::getName(), new StatsUsage()) ->addAction(Migrations::getName(), new Migrations()) ->addAction(StatsResources::getName(), new StatsResources()) - /** Remove */ - ->addAction(UsageDump::getName(), new UsageDump()) - ->addAction(Usage::getName(), new Usage()) - /** /remove */ ; } } diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php deleted file mode 100644 index 3687eeab67..0000000000 --- a/src/Appwrite/Platform/Workers/Usage.php +++ /dev/null @@ -1,292 +0,0 @@ -desc('Usage worker') - ->inject('message') - ->inject('project') - ->inject('getProjectDB') - ->inject('queueForUsageDump') - ->callback(function (Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump) { - $this->action($message, $project, $getProjectDB, $queueForUsageDump); - }); - - $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); - $this->lastTriggeredTime = time(); - } - - /** - * @param Message $message - * @param Document $project - * @param callable $getProjectDB - * @param UsageDump $queueForUsageDump - * @return void - * @throws \Utopia\Database\Exception - * @throws Exception - */ - public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump): void - { - $payload = $message->getPayload() ?? []; - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - - if (empty($project->getAttribute('database'))) { - var_dump($payload); - return; - } - - $projectId = $project->getInternalId(); - foreach ($payload['reduce'] ?? [] as $document) { - if (empty($document)) { - continue; - } - - $this->reduce( - project: $project, - document: new Document($document), - metrics: $payload['metrics'], - getProjectDB: $getProjectDB - ); - } - - - $this->stats[$projectId]['project'] = [ - '$id' => $project->getId(), - '$internalId' => $project->getInternalId(), - 'database' => $project->getAttribute('database'), - ]; - $this->stats[$projectId]['receivedAt'] = DateTime::now(); - foreach ($payload['metrics'] ?? [] as $metric) { - $this->keys++; - if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { - $this->stats[$projectId]['keys'][$metric['key']] = $metric['value']; - continue; - } - - $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; - } - - // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) - if ( - $this->keys >= self::KEYS_THRESHOLD || - (time() - $this->lastTriggeredTime > $this->aggregationInterval && $this->keys > 0) - ) { - Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); - - $queueForUsageDump - ->setStats($this->stats) - ->trigger(); - - $this->stats = []; - $this->keys = 0; - $this->lastTriggeredTime = time(); - } - } - - /** - * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files. - * When we remove a parent document we need to deduct his children aggregation from the project scope. - * @param Document $project - * @param Document $document - * @param array $metrics - * @param callable $getProjectDB - * @return void - */ - private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void - { - $dbForProject = $getProjectDB($project); - - try { - switch (true) { - case $document->getCollection() === 'users': // users - $sessions = count($document->getAttribute(METRIC_SESSIONS, 0)); - if (!empty($sessions)) { - $metrics[] = [ - 'key' => METRIC_SESSIONS, - 'value' => ($sessions * -1), - ]; - } - break; - case $document->getCollection() === 'databases': // databases - $collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS))); - $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS))); - if (!empty($collections['value'])) { - $metrics[] = [ - 'key' => METRIC_COLLECTIONS, - 'value' => ($collections['value'] * -1), - ]; - } - - if (!empty($documents['value'])) { - $metrics[] = [ - 'key' => METRIC_DOCUMENTS, - 'value' => ($documents['value'] * -1), - ]; - } - break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS))); - - if (!empty($documents['value'])) { - $metrics[] = [ - 'key' => METRIC_DOCUMENTS, - 'value' => ($documents['value'] * -1), - ]; - $metrics[] = [ - 'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), - 'value' => ($documents['value'] * -1), - ]; - } - break; - - case $document->getCollection() === 'buckets': - $files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES))); - $storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE))); - - if (!empty($files['value'])) { - $metrics[] = [ - 'key' => METRIC_FILES, - 'value' => ($files['value'] * -1), - ]; - } - - if (!empty($storage['value'])) { - $metrics[] = [ - 'key' => METRIC_FILES_STORAGE, - 'value' => ($storage['value'] * -1), - ]; - } - break; - - case $document->getCollection() === 'functions': - $deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); - $deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); - $builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); - $buildsSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS))); - $buildsFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED))); - $buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); - $buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); - $buildsComputeSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS))); - $buildsComputeFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED))); - $executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); - $executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); - - if (!empty($deployments['value'])) { - $metrics[] = [ - 'key' => METRIC_DEPLOYMENTS, - 'value' => ($deployments['value'] * -1), - ]; - } - - if (!empty($deploymentsStorage['value'])) { - $metrics[] = [ - 'key' => METRIC_DEPLOYMENTS_STORAGE, - 'value' => ($deploymentsStorage['value'] * -1), - ]; - } - - if (!empty($builds['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS, - 'value' => ($builds['value'] * -1), - ]; - } - - if (!empty($buildsSuccess['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_SUCCESS, - 'value' => ($buildsSuccess['value'] * -1), - ]; - } - - if (!empty($buildsFailed['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_FAILED, - 'value' => ($buildsFailed['value'] * -1), - ]; - } - - if (!empty($buildsStorage['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_STORAGE, - 'value' => ($buildsStorage['value'] * -1), - ]; - } - - if (!empty($buildsCompute['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE, - 'value' => ($buildsCompute['value'] * -1), - ]; - } - - if (!empty($buildsComputeSuccess['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE_SUCCESS, - 'value' => ($buildsComputeSuccess['value'] * -1), - ]; - } - - if (!empty($buildsComputeFailed['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE_FAILED, - 'value' => ($buildsComputeFailed['value'] * -1), - ]; - } - - if (!empty($executions['value'])) { - $metrics[] = [ - 'key' => METRIC_EXECUTIONS, - 'value' => ($executions['value'] * -1), - ]; - } - - if (!empty($executionsCompute['value'])) { - $metrics[] = [ - 'key' => METRIC_EXECUTIONS_COMPUTE, - 'value' => ($executionsCompute['value'] * -1), - ]; - } - break; - default: - break; - } - } catch (\Throwable $e) { - console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); - } - } -} diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php deleted file mode 100644 index 2f1d13f29a..0000000000 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ /dev/null @@ -1,286 +0,0 @@ - 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; - - public static function getName(): string - { - return 'usage-dump'; - } - - /** - * @throws \Exception - */ - public function __construct() - { - $this - ->inject('message') - ->inject('getProjectDB') - ->callback(function (Message $message, callable $getProjectDB) { - $this->action($message, $getProjectDB); - }); - } - - /** - * @param Message $message - * @param callable $getProjectDB - * @return void - * @throws Exception - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, callable $getProjectDB): void - { - $payload = $message->getPayload() ?? []; - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - - foreach ($payload['stats'] ?? [] as $stats) { - $project = new Document($stats['project'] ?? []); - - /** - * End temp bug fallback - */ - $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? 'NONE'; - if ($numberOfKeys === 0) { - continue; - } - - console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); - - try { - $dbForProject = $getProjectDB($project); - foreach ($stats['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - if (str_contains($key, METRIC_DATABASES_STORAGE)) { - try { - $this->handleDatabaseStorage($key, $dbForProject); - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); - } - continue; - } - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } - } - } - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); - } - } - } - - private function handleDatabaseStorage(string $key, Database $dbForProject): void - { - $data = explode('.', $key); - $start = microtime(true); - - $updateMetric = function (Database $dbForProject, int $value, string $key, string $period, string|null $time) { - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } - }; - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - $value = 0; - $previousValue = 0; - try { - $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); - } catch (\Exception $e) { - // No previous value - } - - switch (count($data)) { - // Collection Level - case METRIC_COLLECTION_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; - - try { - $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - // Compare with previous value - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Collection - $updateMetric($dbForProject, $diff, $key, $period, $time); - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); - break; - // Database Level - case METRIC_DATABASE_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - - $collections = []; - try { - $collections = $dbForProject->find('database_' . $databaseInternalId); - } catch (\Exception $e) { - // Database not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); - break; - // Project Level - case METRIC_PROJECT_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); - // Get all project databases - $databases = $dbForProject->find('database'); - - // Recalculate all databases - foreach ($databases as $database) { - $collections = $dbForProject->find('database_' . $database->getInternalId()); - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - } - - $diff = $value - $previousValue; - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); - break; - } - } - - $end = microtime(true); - - console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); - } -} From f2fc45da02af88d2e1d69d3e431d78065d6f9a9f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 11 Feb 2025 09:46:00 +0000 Subject: [PATCH 09/65] fix format --- app/worker.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index dc02da1bb7..7fb96f1208 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,8 +16,6 @@ use Appwrite\Event\Migration; use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; /** remove */ -use Appwrite\Event\Usage; -use Appwrite\Event\UsageDump; /** /remove */ use Appwrite\Platform\Appwrite; use Swoole\Runtime; From 721b9b9aea7a025ffbbc6ea950773eaf3eb71ca3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 13 Feb 2025 00:52:20 +0000 Subject: [PATCH 10/65] remove old events --- src/Appwrite/Event/Usage.php | 78 -------------------------------- src/Appwrite/Event/UsageDump.php | 44 ------------------ 2 files changed, 122 deletions(-) delete mode 100644 src/Appwrite/Event/Usage.php delete mode 100644 src/Appwrite/Event/UsageDump.php diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php deleted file mode 100644 index c70cea5c73..0000000000 --- a/src/Appwrite/Event/Usage.php +++ /dev/null @@ -1,78 +0,0 @@ -setQueue(Event::USAGE_QUEUE_NAME) - ->setClass(Event::USAGE_CLASS_NAME); - } - - /** - * Add reduce. - * - * @param Document $document - * @return self - */ - public function addReduce(Document $document): self - { - $this->reduce[] = $document; - - return $this; - } - - /** - * Add metric. - * - * @param string $key - * @param int $value - * @return self - */ - public function addMetric(string $key, int $value): self - { - - $this->metrics[] = [ - 'key' => $key, - 'value' => $value, - ]; - - return $this; - } - - /** - * Prepare the payload for the usage event. - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'project' => $this->project, - 'reduce' => $this->reduce, - 'metrics' => $this->metrics, - ]; - } - - /** - * Sends metrics to the usage worker. - * - * @return string|bool - */ - public function trigger(): string|bool - { - parent::trigger(); - $this->metrics = []; - return true; - } -} diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php deleted file mode 100644 index a70716e94f..0000000000 --- a/src/Appwrite/Event/UsageDump.php +++ /dev/null @@ -1,44 +0,0 @@ -setQueue(Event::USAGE_DUMP_QUEUE_NAME) - ->setClass(Event::USAGE_DUMP_CLASS_NAME); - } - - /** - * Add Stats. - * - * @param array $stats - * @return self - */ - public function setStats(array $stats): self - { - $this->stats = $stats; - - return $this; - } - - /** - * Prepare the payload for the usage dump event. - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'stats' => $this->stats, - ]; - } -} From a74c6e0b10368c16977e576e90fef76be0a9e2cc Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 07:58:56 +0000 Subject: [PATCH 11/65] chore: added targets to realtime, refactor webhook queues --- app/controllers/api/functions.php | 22 ++++----- src/Appwrite/Event/Event.php | 45 +++++++++---------- src/Appwrite/Event/Realtime.php | 38 +++++++++++----- src/Appwrite/Platform/Workers/Builds.php | 30 +++++++------ .../Platform/Workers/Certificates.php | 33 +++++++------- src/Appwrite/Platform/Workers/Databases.php | 2 +- src/Appwrite/Platform/Workers/Functions.php | 22 ++++----- src/Appwrite/Platform/Workers/Migrations.php | 10 +---- 8 files changed, 103 insertions(+), 99 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0f9f5211f4..644aee4336 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -9,6 +9,7 @@ use Appwrite\Event\Func; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\Validator\FunctionEvent; +use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\Validator\Headers; @@ -194,10 +195,12 @@ App::post('/v1/functions') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') + ->inject('queueForWebhooks') + ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Realtime $queueForRealtime, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check @@ -396,26 +399,19 @@ App::post('/v1/functions') ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); /** Trigger Webhook */ - $ruleCreate - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setQueue(Event::WEBHOOK_QUEUE_NAME) + $queueForWebhooks + ->from($ruleCreate) ->trigger(); /** Trigger Functions */ - $ruleCreate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + $queueForFunctions + ->from($ruleCreate) ->trigger(); /** Trigger Realtime Events */ $queueForRealtime ->from($ruleCreate) - ->setProjectId('console') - ->trigger(); - - $queueForRealtime - ->from($ruleCreate) - ->setProjectId($project->getId()) + ->setTargets(['console', $project->getId()]) ->trigger(); } diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 8085a836a8..2187210e34 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -62,11 +62,11 @@ class Event protected array $params = []; protected array $sensitive = []; protected array $payload = []; + protected array $targets = []; protected array $context = []; protected ?Document $project = null; protected ?Document $user = null; protected ?string $userId = null; - protected ?string $projectId = null; protected bool $paused = false; /** @@ -152,18 +152,6 @@ class Event return $this; } - /** - * Set projectId for this event. - * - * @param string $projectId - * @return self - */ - public function setProjectId(string $projectId): self - { - $this->projectId = $projectId; - return $this; - } - /** * Get project for this event. * @@ -174,16 +162,6 @@ class Event return $this->project; } - /** - * Get projectId for this event. - * - * @return ?string - */ - public function getProjectId(): ?string - { - return $this->projectId; - } - /** * Set user for this event. * @@ -255,6 +233,27 @@ class Event return $this->payload; } + /** + * Get targets for this event. + * + * @return array + */ + public function setTargets(array $targets): self + { + $this->targets = $targets; + return $this; + } + + /** + * Get targets for this event. + * + * @return array + */ + public function getTargets(): array + { + return $this->targets; + } + /** * Set context for this event. * diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index 8c302bbabf..d16057ad2c 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -53,17 +53,33 @@ class Realtime extends Event bucket: $bucket, ); - RealtimeAdapter::send( - projectId: $this->getProjectId() ?? $target['projectId'] ?? $this->getProject()->getId(), - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); + if (!empty($this->getTargets())) { + foreach ($this->getTargets() as $targetProjectId) { + RealtimeAdapter::send( + projectId: $targetProjectId, + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); + } + } else { + RealtimeAdapter::send( + projectId: $target['projectId'] ?? $this->getProject()->getId(), + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); + } return true; } diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 0bc0937da0..8120adccb6 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -7,6 +7,7 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; +use Appwrite\Event\Webhook; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -49,6 +50,7 @@ class Builds extends Action ->inject('project') ->inject('dbForPlatform') ->inject('queueForEvents') + ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('queueForStatsUsage') @@ -57,8 +59,8 @@ class Builds extends Action ->inject('deviceForFunctions') ->inject('isResourceBlocked') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => - $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => + $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); } /** @@ -66,6 +68,7 @@ class Builds extends Action * @param Document $project * @param Database $dbForPlatform * @param Event $queueForEvents + * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime * @param StatsUsage $queueForStatsUsage @@ -76,7 +79,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void { $payload = $message->getPayload() ?? []; @@ -97,7 +100,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); + $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); break; default: @@ -107,6 +110,7 @@ class Builds extends Action /** * @param Device $deviceForFunctions + * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime * @param Event $queueForEvents @@ -123,7 +127,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Webhooks $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -379,7 +383,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -431,19 +435,19 @@ class Builds extends Action $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); } - /** Trigger Webhook */ $deploymentModel = new Deployment(); $deploymentUpdate = $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) ->setProject($project) ->setEvent('functions.[functionId].deployments.[deploymentId].update') ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); - $deploymentUpdate->trigger(); + /** Trigger Webhook */ + $queueForWebhooks + ->from($deploymentUpdate) + ->trigger(); /** Trigger Functions */ $queueForFunctions @@ -453,7 +457,7 @@ class Builds extends Action /** Trigger Realtime Event */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -585,7 +589,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -682,7 +686,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index c7f345f77a..009ac24021 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -7,6 +7,7 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Realtime; +use Appwrite\Event\Webhook; use Appwrite\Network\Validator\CNAME; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; @@ -46,13 +47,14 @@ class Certificates extends Action ->inject('dbForPlatform') ->inject('queueForMails') ->inject('queueForEvents') + ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('log') ->inject('certificates') ->callback( - fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates) => - $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates) + fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates) ); } @@ -61,6 +63,7 @@ class Certificates extends Action * @param Database $dbForPlatform * @param Mail $queueForMails * @param Event $queueForEvents + * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime * @param Log $log @@ -69,7 +72,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates): void + public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -83,7 +86,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck); + $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck); } /** @@ -99,7 +102,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -189,7 +192,7 @@ class Certificates extends Action $certificate->setAttribute('updated', DateTime::now()); // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); + $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } @@ -209,7 +212,7 @@ class Certificates extends Action * @throws Conflict * @throws Structure */ - private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void + private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime): void { // Check if update or insert required $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); @@ -223,7 +226,7 @@ class Certificates extends Action } $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); + $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } /** @@ -342,7 +345,7 @@ class Certificates extends Action * * @return void */ - private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void + private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime): void { // TODO: @christyjacob remove once we migrate the rules in 1.7.x if (System::getEnv('_APP_RULES_FORMAT') === 'md5') { @@ -379,9 +382,8 @@ class Certificates extends Action ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); /** Trigger Webhook */ - $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) + $queueForWebhooks + ->from($queueForEvents) ->trigger(); /** Trigger Functions */ @@ -392,12 +394,7 @@ class Certificates extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setProjectId('console') - ->trigger(); - - $queueForRealtime - ->from($queueForEvents) - ->setProjectId($project->getId()) + ->setTargets(['console', $projectId]) ->trigger(); } } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 7d58521b82..ae75a74deb 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -604,7 +604,7 @@ class Databases extends Action ): void { $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('databaseId', $database->getId()) ->setParam('collectionId', $collection->getId()); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index f485db8648..73b02be85e 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -7,6 +7,7 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; +use Appwrite\Event\Webhook; use Appwrite\Utopia\Response\Model\Execution; use Exception; use Executor\Executor; @@ -44,16 +45,17 @@ class Functions extends Action ->inject('project') ->inject('message') ->inject('dbForProject') + ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('log') ->inject('isResourceBlocked') - ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); + ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); } - public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void + public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void { $payload = $message->getPayload() ?? []; @@ -137,6 +139,7 @@ class Functions extends Action $this->execute( log: $log, dbForProject: $dbForProject, + queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, @@ -178,6 +181,7 @@ class Functions extends Action $this->execute( log: $log, dbForProject: $dbForProject, + queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, @@ -201,6 +205,7 @@ class Functions extends Action $this->execute( log: $log, dbForProject: $dbForProject, + queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, @@ -312,6 +317,7 @@ class Functions extends Action private function execute( Log $log, Database $dbForProject, + Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, @@ -582,9 +588,8 @@ class Functions extends Action ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))); /** Trigger Webhook */ - $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) + $queueForWebhooks + ->from($queueForEvents) ->trigger(); /** Trigger Functions */ @@ -595,12 +600,7 @@ class Functions extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setProjectId('console') - ->trigger(); - - $queueForRealtime - ->from($queueForEvents) - ->setProjectId($project->getId()) + ->setTargets(['console', $project->getId()]) ->trigger(); if (!empty($error)) { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 50d6002d28..541c171a22 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -160,15 +160,7 @@ class Migrations extends Action /** Trigger Realtime Events */ $queueForRealtime ->setProject($project) - ->setProjectId('console') - ->setEvent('migrations.[migrationId].update') - ->setParam('migrationId', $migration->getId()) - ->setPayload($migration->getArrayCopy()) - ->trigger(); - - $queueForRealtime - ->setProject($project) - ->setProjectId($project->getId()) + ->setTargets(['console', $project->getId()]) ->setEvent('migrations.[migrationId].update') ->setParam('migrationId', $migration->getId()) ->setPayload($migration->getArrayCopy()) From 235a357ca3aa22876881840d262ca34853473331 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 11:37:07 +0000 Subject: [PATCH 12/65] chore: initialized queueForWebhooks in worker init --- app/worker.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/worker.php b/app/worker.php index ad6bf475f9..1088a9b6db 100644 --- a/app/worker.php +++ b/app/worker.php @@ -20,6 +20,7 @@ use Appwrite\Event\StatsUsageDump; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; /** /remove */ +use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; @@ -314,6 +315,10 @@ Server::setResource('queueForAudits', function (Publisher $publisher) { return new Audit($publisher); }, ['publisher']); +Server::setResource('queueForWebhooks', function (Publisher $publisher) { + return new Webhook($publisher); +}, ['publisher']); + Server::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); From 9f7aaf702913548ce52cc7311973f0ba7de97f03 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 12:51:19 +0000 Subject: [PATCH 13/65] chore: fix warning in database worker --- src/Appwrite/Event/Event.php | 3 ++- src/Appwrite/Platform/Workers/Databases.php | 28 ++++++++++----------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 2187210e34..7e15cabca9 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -234,8 +234,9 @@ class Event } /** - * Get targets for this event. + * Set targets for this event. * + * @param array $targets * @return array */ public function setTargets(array $targets): self diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index ae75a74deb..b85285ae72 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -197,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, $attribute); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -304,7 +304,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, $attribute); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -418,7 +418,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -474,7 +474,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -586,21 +586,19 @@ class Databases extends Action * @param Document $database * @param Document $collection * @param Document $project + * @param Realtime $queueForRealtime * @param Document|null $attribute * @param Document|null $index - * @param Document $payload - * @param Realtime $queueForRealtime * @return void */ protected function trigger( Document $database, Document $collection, Document $project, + string $event, + Realtime $queueForRealtime, Document|null $attribute = null, Document|null $index = null, - Document $payload, - string $event, - Realtime $queueForRealtime ): void { $queueForRealtime ->setProject($project) @@ -610,14 +608,16 @@ class Databases extends Action ->setParam('collectionId', $collection->getId()); if ($attribute !== null) { - $queueForRealtime->setParam('attributeId', $attribute->getId()); + $queueForRealtime + ->setParam('attributeId', $attribute->getId()) + ->setPayload($attribute->getArrayCopy()); } if ($index !== null) { - $queueForRealtime->setParam('indexId', $index->getId()); + $queueForRealtime + ->setParam('indexId', $index->getId()) + ->setPayload($index->getArrayCopy()); } - $queueForRealtime - ->setPayload($payload->getArrayCopy()) - ->trigger(); + $queueForRealtime->trigger(); } } From 88a6616cf8572a6a1132b1527dfde654cbb8a819 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 13:05:50 +0000 Subject: [PATCH 14/65] chore: fix naming --- src/Appwrite/Platform/Workers/Builds.php | 2 +- src/Appwrite/Platform/Workers/Databases.php | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 8120adccb6..c3853a1dde 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -127,7 +127,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Webhooks $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index b85285ae72..74101dd1eb 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -516,7 +516,6 @@ class Databases extends Action $collectionId = $collection->getId(); $collectionInternalId = $collection->getInternalId(); - $databaseId = $database->getId(); $databaseInternalId = $database->getInternalId(); $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); From 55e7633bd12c2631f03f9bc75e5967e1b600f0cd Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 14:04:59 +0000 Subject: [PATCH 15/65] chore: shift targets to realtime --- src/Appwrite/Event/Event.php | 23 ----------------------- src/Appwrite/Event/Realtime.php | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 7e15cabca9..0edffdf4dc 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -62,7 +62,6 @@ class Event protected array $params = []; protected array $sensitive = []; protected array $payload = []; - protected array $targets = []; protected array $context = []; protected ?Document $project = null; protected ?Document $user = null; @@ -233,28 +232,6 @@ class Event return $this->payload; } - /** - * Set targets for this event. - * - * @param array $targets - * @return array - */ - public function setTargets(array $targets): self - { - $this->targets = $targets; - return $this; - } - - /** - * Get targets for this event. - * - * @return array - */ - public function getTargets(): array - { - return $this->targets; - } - /** * Set context for this event. * diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index d16057ad2c..e04bbfffb4 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -7,10 +7,17 @@ use Utopia\Database\Document; class Realtime extends Event { + protected array $targets = []; + public function __construct() { } + /** + * Get Realtime payload for this event. + * + * @return array + */ public function getRealtimePayload(): array { $payload = []; @@ -24,6 +31,28 @@ class Realtime extends Event return $payload; } + /** + * Set targets for this realtime event. + * + * @param array $targets + * @return array + */ + public function setTargets(array $targets): self + { + $this->targets = $targets; + return $this; + } + + /** + * Get targets for this realtime event. + * + * @return array + */ + public function getTargets(): array + { + return $this->targets; + } + /** * Execute Event. * From 6235a1c729627302bb991afdf419906abb848b99 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 10:33:58 +0000 Subject: [PATCH 16/65] chore: make min/max params optional for attribute update --- app/controllers/api/databases.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c563d1e8e0..a4b5683512 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2339,14 +2339,19 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') ->param('required', null, new Boolean(), 'Is attribute required?') - ->param('min', null, new Integer(), 'Minimum value to enforce on new documents') - ->param('max', null, new Integer(), 'Maximum value to enforce on new documents') + ->param('min', null, new Integer(), 'Minimum value to enforce on new documents', true) + ->param('max', null, new Integer(), 'Maximum value to enforce on new documents', true) ->param('default', null, new Nullable(new Integer()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->param('newKey', null, new Key(), 'New attribute key.', true) ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { + + // Ensure attribute default is within range + $min = \is_null($min) ? PHP_INT_MIN : $min; + $max = \is_null($max) ? PHP_INT_MAX : $max; + $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, @@ -2398,14 +2403,19 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') ->param('required', null, new Boolean(), 'Is attribute required?') - ->param('min', null, new FloatValidator(), 'Minimum value to enforce on new documents') - ->param('max', null, new FloatValidator(), 'Maximum value to enforce on new documents') + ->param('min', null, new FloatValidator(), 'Minimum value to enforce on new documents', true) + ->param('max', null, new FloatValidator(), 'Maximum value to enforce on new documents', true) ->param('default', null, new Nullable(new FloatValidator()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->param('newKey', null, new Key(), 'New attribute key.', true) ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { + + // Ensure attribute default is within range + $min = \is_null($min) ? -PHP_FLOAT_MAX : $min; + $max = \is_null($max) ? PHP_FLOAT_MAX : $max; + $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, From ccb7dfa839311a78c624c516800ed1cc0d142a53 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 11:15:05 +0000 Subject: [PATCH 17/65] chore: added test cases --- .../e2e/Services/Databases/DatabasesBase.php | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 0f57f94515..6a4839274c 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -464,6 +464,38 @@ trait DatabasesBase $this->assertEquals(400, $attribute['headers']['status-code']); $this->assertStringContainsString('Index length is longer than the maximum: 76', $attribute['body']['message']); + + $integer = $this->client->call(Client::METHOD_POST, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'integer', + 'required' => false, + 'min' => 1, + 'max' => 5, + 'default' => 3 + ]); + $this->assertEquals(202, $integer['headers']['status-code']); + + sleep(1); + + /** + * Update integer default value to 10 + */ + $integer = $this->client->call(Client::METHOD_PATCH, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer/'.$integer['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]), [ + 'key' => 'integer', + 'required' => false, + 'default' => 10 + ]); + + $this->assertEquals(200, $integer['headers']['status-code']); + $this->assertEquals(PHP_INT_MIN, $integer['body']['min']); + $this->assertEquals(PHP_INT_MAX, $integer['body']['max']); } public function testUpdateAttributeEnum(): void From b0ced9b351746a23c4fd976f6ff0e022e459a541 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 13:37:13 +0000 Subject: [PATCH 18/65] chore: updated tests --- .../e2e/Services/Databases/DatabasesBase.php | 32 ------ .../Databases/DatabasesCustomServerTest.php | 102 +++++++++--------- 2 files changed, 50 insertions(+), 84 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 6a4839274c..0f57f94515 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -464,38 +464,6 @@ trait DatabasesBase $this->assertEquals(400, $attribute['headers']['status-code']); $this->assertStringContainsString('Index length is longer than the maximum: 76', $attribute['body']['message']); - - $integer = $this->client->call(Client::METHOD_POST, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'key' => 'integer', - 'required' => false, - 'min' => 1, - 'max' => 5, - 'default' => 3 - ]); - $this->assertEquals(202, $integer['headers']['status-code']); - - sleep(1); - - /** - * Update integer default value to 10 - */ - $integer = $this->client->call(Client::METHOD_PATCH, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer/'.$integer['body']['key'], array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]), [ - 'key' => 'integer', - 'required' => false, - 'default' => 10 - ]); - - $this->assertEquals(200, $integer['headers']['status-code']); - $this->assertEquals(PHP_INT_MIN, $integer['body']['min']); - $this->assertEquals(PHP_INT_MAX, $integer['body']['max']); } public function testUpdateAttributeEnum(): void diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index b501e2119e..ac1df82c9d 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2309,6 +2309,30 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(100, $new['body']['min']); $this->assertEquals(2000, $new['body']['max']); + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 100, + 'min' => 0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 100, + 'max' => 0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + /** * Test against failure */ @@ -2368,32 +2392,6 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $update['headers']['status-code']); $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 100, - 'min' => 0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 100, - 'max' => 0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -2572,6 +2570,32 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(123.456, $new['body']['min']); $this->assertEquals(2000, $new['body']['max']); + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 123.456, + 'min' => 0.0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); + + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 123.456, + 'max' => 0.0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); + /** * Test against failure */ @@ -2631,32 +2655,6 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $update['headers']['status-code']); $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 123.456, - 'min' => 0.0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 123.456, - 'max' => 0.0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], From 077fe10d808a04c2a500c6d560633b5163375429 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 14:12:14 +0000 Subject: [PATCH 19/65] chore: fix tests --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index ac1df82c9d..213584c3df 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2327,7 +2327,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => 100, + 'default' => -10, 'max' => 0, ]); @@ -2589,7 +2589,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => 123.456, + 'default' => -123.456, 'max' => 0.0, ]); From f9cc7882aa78e3ab6e66a202afdf0ebfcbbe3f5c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 14:43:21 +0000 Subject: [PATCH 20/65] chore: fix tests --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 213584c3df..ad581eecb6 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2581,7 +2581,6 @@ class DatabasesCustomServerTest extends Scope ]); $this->assertEquals(200, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ 'content-type' => 'application/json', From c660b9d858b9cb9b0e2ee85f8e5f64d30d555cb5 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 14:56:23 +0000 Subject: [PATCH 21/65] chore: fix tests --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index ad581eecb6..75526b9928 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2593,7 +2593,6 @@ class DatabasesCustomServerTest extends Scope ]); $this->assertEquals(200, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); /** * Test against failure From b1629b24ad2fbde59a3867ae061e307f28ef2504 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Feb 2025 11:44:04 +0000 Subject: [PATCH 22/65] chore: added file transformation stats to usage endpoint for bucket --- app/controllers/api/storage.php | 3 +++ src/Appwrite/Utopia/Response/Model/UsageBuckets.php | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f180c22acf..efd82ba40e 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1894,6 +1894,7 @@ App::get('/v1/storage/:bucketId/usage') $metrics = [ str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_TRANSFORMATIONS), ]; @@ -1948,5 +1949,7 @@ App::get('/v1/storage/:bucketId/usage') 'filesStorageTotal' => $usage[$metrics[1]]['total'], 'files' => $usage[$metrics[0]]['data'], 'storage' => $usage[$metrics[1]]['data'], + 'fileTransformations' => $usage[$metrics[2]]['data'], + 'fileTransformationsTotal' => $usage[$metrics[2]]['total'], ]), Response::MODEL_USAGE_BUCKETS); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 2f528ac9d1..ab8d129857 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -42,6 +42,19 @@ class UsageBuckets extends Model 'example' => [], 'array' => true ]) + ->addRule('fileTransformations', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of files transformations per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('fileTransformationsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of files transformations.', + 'default' => 0, + 'example' => 0, + ]) ; } From 4ed4c660d5aff0ee10cbdaad8ff2e15293b3ed88 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Feb 2025 12:53:17 +0000 Subject: [PATCH 23/65] chore: added tests for file transformations --- tests/e2e/Services/Storage/StorageConsoleClientTest.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index 5b6731b35e..b3a6ad9274 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -104,5 +104,7 @@ class StorageConsoleClientTest extends Scope $this->assertIsNumeric($response['body']['filesStorageTotal']); $this->assertIsArray($response['body']['files']); $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['fileTransformations']); + $this->assertIsNumeric($response['body']['fileTransformationsTotal']); } } From 0e0bab4c40a54a310b9d07599372ec8be8d06ac1 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Feb 2025 13:28:49 +0000 Subject: [PATCH 24/65] chore: updated test --- tests/e2e/Services/Storage/StorageConsoleClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b3a6ad9274..b2624746ca 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -98,7 +98,7 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(5, count($response['body'])); + $this->assertEquals(7, count($response['body'])); $this->assertEquals('24h', $response['body']['range']); $this->assertIsNumeric($response['body']['filesTotal']); $this->assertIsNumeric($response['body']['filesStorageTotal']); From 7ac4a5bf271b7079210996c01b39bc186aa4f207 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 25 Feb 2025 04:43:44 +0000 Subject: [PATCH 25/65] chore: update naming --- app/controllers/api/storage.php | 6 +++--- src/Appwrite/Utopia/Response/Model/UsageBuckets.php | 4 ++-- tests/e2e/Services/Storage/StorageConsoleClientTest.php | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index efd82ba40e..072a2d4ae4 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1894,7 +1894,7 @@ App::get('/v1/storage/:bucketId/usage') $metrics = [ str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), - str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_TRANSFORMATIONS), + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_IMAGES_TRANSFORMED), ]; @@ -1949,7 +1949,7 @@ App::get('/v1/storage/:bucketId/usage') 'filesStorageTotal' => $usage[$metrics[1]]['total'], 'files' => $usage[$metrics[0]]['data'], 'storage' => $usage[$metrics[1]]['data'], - 'fileTransformations' => $usage[$metrics[2]]['data'], - 'fileTransformationsTotal' => $usage[$metrics[2]]['total'], + 'imageTransformations' => $usage[$metrics[2]]['data'], + 'imageTransformationsTotal' => $usage[$metrics[2]]['total'], ]), Response::MODEL_USAGE_BUCKETS); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index ab8d129857..ee624a29ad 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -42,14 +42,14 @@ class UsageBuckets extends Model 'example' => [], 'array' => true ]) - ->addRule('fileTransformations', [ + ->addRule('imageTransformations', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of files transformations per period.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('fileTransformationsTotal', [ + ->addRule('imageTransformationsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of files transformations.', 'default' => 0, diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b2624746ca..bbb14fb136 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -104,7 +104,7 @@ class StorageConsoleClientTest extends Scope $this->assertIsNumeric($response['body']['filesStorageTotal']); $this->assertIsArray($response['body']['files']); $this->assertIsArray($response['body']['storage']); - $this->assertIsArray($response['body']['fileTransformations']); - $this->assertIsNumeric($response['body']['fileTransformationsTotal']); + $this->assertIsArray($response['body']['imageTransformations']); + $this->assertIsNumeric($response['body']['imageTransformationsTotal']); } } From 676431c355d93597d34853b95c286d9bd73525e3 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 25 Feb 2025 07:16:42 +0000 Subject: [PATCH 26/65] chore: shifted to logsDb --- app/controllers/api/storage.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 072a2d4ae4..972232167b 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1879,9 +1879,12 @@ App::get('/v1/storage/:bucketId/usage') ->param('bucketId', '', new UID(), 'Bucket ID.') ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) ->inject('response') + ->inject('project') ->inject('dbForProject') - ->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) { + ->inject('getLogsDB') + ->action(function (string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, callable $getLogsDB) { + $dbForLogs = call_user_func($getLogsDB, $project); $bucket = $dbForProject->getDocument('buckets', $bucketId); if ($bucket->isEmpty()) { @@ -1894,13 +1897,16 @@ App::get('/v1/storage/:bucketId/usage') $metrics = [ str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), - str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_IMAGES_TRANSFORMED), + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED), ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { + Authorization::skip(function () use ($dbForProject, $dbForLogs, $bucket, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $db = ($metric === str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED)) + ? $dbForLogs + : $dbForProject; + + $result = $db->findOne('stats', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -1908,7 +1914,7 @@ App::get('/v1/storage/:bucketId/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $db->find('stats', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), From ae78e793dd3d026e376def042d2b88fbec9ecd80 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 08:52:06 +0000 Subject: [PATCH 27/65] chore: added blocking to file preview endpoint --- app/config/errors.php | 5 +++++ app/controllers/api/storage.php | 18 ++++++++++-------- src/Appwrite/Extend/Exception.php | 1 + 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index 7c7f6dc9ec..d8a5c798d3 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -480,6 +480,11 @@ return [ 'description' => 'The requested file is not publicly readable.', 'code' => 403, ], + Exception::STORAGE_FILE_PREVIEW_BLOCKED => [ + 'name' => Exception::STORAGE_FILE_PREVIEW_BLOCKED, + 'description' => 'File preview is not available on your pricing current tier.', + 'code' => 403, + ], /** VCS */ Exception::INSTALLATION_NOT_FOUND => [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 972232167b..38e74ad2ee 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -935,15 +935,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->param('rotation', 0, new Range(-360, 360), 'Preview image rotation in degrees. Pass an integer between -360 and 360.', true) ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) - ->inject('request') ->inject('response') - ->inject('project') + ->inject('plan') ->inject('dbForProject') - ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') ->inject('queueForStatsUsage') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Response $response, array $plan, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -965,6 +963,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception(Exception::USER_UNAUTHORIZED); } + if (isset($plan['imageTransformations'])) { + if ($plan['imageTransformations'] === -1) { + throw new Exception(Exception::STORAGE_FILE_PREVIEW_BLOCKED); + } + } + if ($fileSecurity && !$valid) { $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } else { @@ -1073,8 +1077,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $queueForStatsUsage ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1); $transformedAt = $file->getAttribute('transformedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { @@ -1085,8 +1088,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days ->setContentType($contentType) - ->file($data) - ; + ->file($data); unset($image); }); diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index d4f47ca177..7f601018fe 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -143,6 +143,7 @@ class Exception extends \Exception public const STORAGE_INVALID_RANGE = 'storage_invalid_range'; public const STORAGE_INVALID_APPWRITE_ID = 'storage_invalid_appwrite_id'; public const STORAGE_FILE_NOT_PUBLIC = 'storage_file_not_public'; + public const STORAGE_FILE_PREVIEW_BLOCKED = 'storage_file_preview_blocked'; /** VCS */ public const INSTALLATION_NOT_FOUND = 'installation_not_found'; From 0ddcb9b14c9a07a647a5eb3e5bf95f41b3e2b891 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 16:24:05 +0000 Subject: [PATCH 28/65] chore: refactor adapter --- src/Appwrite/Event/Realtime.php | 42 ++++++++++++--------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index e04bbfffb4..e7a61ea545 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -82,33 +82,21 @@ class Realtime extends Event bucket: $bucket, ); - if (!empty($this->getTargets())) { - foreach ($this->getTargets() as $targetProjectId) { - RealtimeAdapter::send( - projectId: $targetProjectId, - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); - } - } else { - RealtimeAdapter::send( - projectId: $target['projectId'] ?? $this->getProject()->getId(), - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); - } + $projectIds = !empty($this->getTargets()) + ? $this->getTargets() + : [$target['projectId'] ?? $this->getProject()->getId()]; + + RealtimeAdapter::send( + projectId: $projectIds, + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); return true; } From 7687784515ab4e0f78cf8f7d024281a968dff341 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 16:41:26 +0000 Subject: [PATCH 29/65] chore: added fallback to old values in update --- app/controllers/api/databases.php | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a4b5683512..e14c504234 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -299,6 +299,9 @@ function updateAttribute( switch ($attribute->getAttribute('format')) { case APP_DATABASE_ATTRIBUTE_INT_RANGE: case APP_DATABASE_ATTRIBUTE_FLOAT_RANGE: + $min ??= $attribute->getAttribute('formatOptions')['min']; + $max ??= $attribute->getAttribute('formatOptions')['max']; + if ($min > $max) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value'); } @@ -1550,8 +1553,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // Ensure attribute default is within range - $min = \is_null($min) ? PHP_INT_MIN : $min; - $max = \is_null($max) ? PHP_INT_MAX : $max; + $min ??= PHP_INT_MIN; + $max ??= PHP_INT_MAX; if ($min > $max) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value'); @@ -1627,8 +1630,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float' ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // Ensure attribute default is within range - $min = \is_null($min) ? -PHP_FLOAT_MAX : $min; - $max = \is_null($max) ? PHP_FLOAT_MAX : $max; + $min ??= -PHP_FLOAT_MAX; + $max ??= PHP_FLOAT_MAX; if ($min > $max) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value'); @@ -2347,11 +2350,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { - - // Ensure attribute default is within range - $min = \is_null($min) ? PHP_INT_MIN : $min; - $max = \is_null($max) ? PHP_INT_MAX : $max; - $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, @@ -2411,11 +2409,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { - - // Ensure attribute default is within range - $min = \is_null($min) ? -PHP_FLOAT_MAX : $min; - $max = \is_null($max) ? PHP_FLOAT_MAX : $max; - $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, From 28896081d78cf12a0a7c06838f7fb2987f823bb1 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 16:50:12 +0000 Subject: [PATCH 30/65] chore: fix forloop --- src/Appwrite/Event/Realtime.php | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index e7a61ea545..324f140165 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -86,17 +86,19 @@ class Realtime extends Event ? $this->getTargets() : [$target['projectId'] ?? $this->getProject()->getId()]; - RealtimeAdapter::send( - projectId: $projectIds, - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); + foreach ($projectIds as $projectId) { + RealtimeAdapter::send( + projectId: $projectId, + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); + } return true; } From c79ed286b1cc79ba94778acda18983e8052da288 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 17:35:06 +0000 Subject: [PATCH 31/65] chore: update tests --- app/controllers/api/databases.php | 4 ++-- .../e2e/Services/Databases/DatabasesCustomServerTest.php | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index e14c504234..876c6bc82a 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -242,8 +242,8 @@ function updateAttribute( string $filter = null, string|bool|int|float $default = null, bool $required = null, - int|float $min = null, - int|float $max = null, + int|float|null $min = null, + int|float|null $max = null, array $elements = null, array $options = [], string $newKey = null, diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 75526b9928..57e0b93634 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2327,8 +2327,8 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => -10, - 'max' => 0, + 'default' => 10, + 'max' => 100, ]); $this->assertEquals(200, $update['headers']['status-code']); @@ -2588,8 +2588,8 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => -123.456, - 'max' => 0.0, + 'default' => 23.456, + 'max' => 100.0, ]); $this->assertEquals(200, $update['headers']['status-code']); From 388affa3d0f9f0c231f731b30b86cc4127d2c129 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 27 Feb 2025 07:23:13 +0000 Subject: [PATCH 32/65] chore: updated naming for subscribers --- app/controllers/api/functions.php | 2 +- src/Appwrite/Event/Realtime.php | 20 +++++++++---------- src/Appwrite/Platform/Workers/Builds.php | 8 ++++---- .../Platform/Workers/Certificates.php | 2 +- src/Appwrite/Platform/Workers/Databases.php | 6 +++--- src/Appwrite/Platform/Workers/Functions.php | 2 +- src/Appwrite/Platform/Workers/Migrations.php | 2 +- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a36313b7ab..583468f6c1 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -411,7 +411,7 @@ App::post('/v1/functions') /** Trigger Realtime Events */ $queueForRealtime ->from($ruleCreate) - ->setTargets(['console', $project->getId()]) + ->setSubscribers(['console', $project->getId()]) ->trigger(); } diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index 324f140165..28a1bb6a6d 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -7,7 +7,7 @@ use Utopia\Database\Document; class Realtime extends Event { - protected array $targets = []; + protected array $subscribers = []; public function __construct() { @@ -32,25 +32,25 @@ class Realtime extends Event } /** - * Set targets for this realtime event. + * Set subscribers for this realtime event. * - * @param array $targets + * @param array $subscribers * @return array */ - public function setTargets(array $targets): self + public function setSubscribers(array $subscribers): self { - $this->targets = $targets; + $this->subscribers = $subscribers; return $this; } /** - * Get targets for this realtime event. + * Get subscribers for this realtime event. * * @return array */ - public function getTargets(): array + public function getSubscribers(): array { - return $this->targets; + return $this->subscribers; } /** @@ -82,8 +82,8 @@ class Realtime extends Event bucket: $bucket, ); - $projectIds = !empty($this->getTargets()) - ? $this->getTargets() + $projectIds = !empty($this->getSubscribers()) + ? $this->getSubscribers() : [$target['projectId'] ?? $this->getProject()->getId()]; foreach ($projectIds as $projectId) { diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index c3853a1dde..6f26e9a80c 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -383,7 +383,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -457,7 +457,7 @@ class Builds extends Action /** Trigger Realtime Event */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -589,7 +589,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -686,7 +686,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 009ac24021..093e6fda5a 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -394,7 +394,7 @@ class Certificates extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setTargets(['console', $projectId]) + ->setSubscribers(['console', $projectId]) ->trigger(); } } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 74101dd1eb..4abd035599 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -601,17 +601,17 @@ class Databases extends Action ): void { $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('databaseId', $database->getId()) ->setParam('collectionId', $collection->getId()); - if ($attribute !== null) { + if ($attribute !== null && !empty($attribute)) { $queueForRealtime ->setParam('attributeId', $attribute->getId()) ->setPayload($attribute->getArrayCopy()); } - if ($index !== null) { + if ($index !== null && !empty($index)) { $queueForRealtime ->setParam('indexId', $index->getId()) ->setPayload($index->getArrayCopy()); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 73b02be85e..a7caa3207f 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -600,7 +600,7 @@ class Functions extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setTargets(['console', $project->getId()]) + ->setSubscribers(['console', $project->getId()]) ->trigger(); if (!empty($error)) { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 08c75d934a..f21a846a0d 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -160,7 +160,7 @@ class Migrations extends Action /** Trigger Realtime Events */ $queueForRealtime ->setProject($project) - ->setTargets(['console', $project->getId()]) + ->setSubscribers(['console', $project->getId()]) ->setEvent('migrations.[migrationId].update') ->setParam('migrationId', $migration->getId()) ->setPayload($migration->getArrayCopy()) From f31acd214ec73385d1f7668ed80787dfd4be85ba Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 27 Feb 2025 09:52:27 +0000 Subject: [PATCH 33/65] chore: remove plan blocking --- app/config/errors.php | 5 ----- app/controllers/api/storage.php | 9 +-------- src/Appwrite/Extend/Exception.php | 1 - 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index d8a5c798d3..7c7f6dc9ec 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -480,11 +480,6 @@ return [ 'description' => 'The requested file is not publicly readable.', 'code' => 403, ], - Exception::STORAGE_FILE_PREVIEW_BLOCKED => [ - 'name' => Exception::STORAGE_FILE_PREVIEW_BLOCKED, - 'description' => 'File preview is not available on your pricing current tier.', - 'code' => 403, - ], /** VCS */ Exception::INSTALLATION_NOT_FOUND => [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 38e74ad2ee..bee245b64f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -936,12 +936,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) ->inject('response') - ->inject('plan') ->inject('dbForProject') ->inject('deviceForFiles') ->inject('deviceForLocal') ->inject('queueForStatsUsage') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Response $response, array $plan, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Response $response, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -963,12 +962,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception(Exception::USER_UNAUTHORIZED); } - if (isset($plan['imageTransformations'])) { - if ($plan['imageTransformations'] === -1) { - throw new Exception(Exception::STORAGE_FILE_PREVIEW_BLOCKED); - } - } - if ($fileSecurity && !$valid) { $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } else { diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 7f601018fe..d4f47ca177 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -143,7 +143,6 @@ class Exception extends \Exception public const STORAGE_INVALID_RANGE = 'storage_invalid_range'; public const STORAGE_INVALID_APPWRITE_ID = 'storage_invalid_appwrite_id'; public const STORAGE_FILE_NOT_PUBLIC = 'storage_file_not_public'; - public const STORAGE_FILE_PREVIEW_BLOCKED = 'storage_file_preview_blocked'; /** VCS */ public const INSTALLATION_NOT_FOUND = 'installation_not_found'; From e9d8d13fbdf79dfe03fd6bd61474348d1f877e01 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 08:28:21 +0000 Subject: [PATCH 34/65] chore: added imageTransformations to project usage --- app/controllers/api/project.php | 2 ++ src/Appwrite/Utopia/Response/Model/UsageProject.php | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index ea2cd4436d..cf2a128939 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -63,6 +63,7 @@ App::get('/v1/project/usage') METRIC_BUILDS_STORAGE, METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, + METRIC_FILES_IMAGES_TRANSFORMED, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -363,6 +364,7 @@ App::get('/v1/project/usage') 'authPhoneTotal' => $authPhoneTotal, 'authPhoneEstimate' => $authPhoneEstimate, 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, + 'imageTransformations' => $total[METRIC_FILES_IMAGES_TRANSFORMED], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 1006276b56..6f74ec561b 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -197,6 +197,13 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('imageTransformations', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of image transformations.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } From d5cfdc570f25dbab1a591df1f8e32e800fe216fa Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 08:41:53 +0000 Subject: [PATCH 35/65] chore: updated specs --- app/config/specs/open-api3-latest-client.json | 10 +- .../specs/open-api3-latest-console.json | 211 ++++++++++------- app/config/specs/open-api3-latest-server.json | 178 ++++++++------- app/config/specs/swagger2-latest-client.json | 10 +- app/config/specs/swagger2-latest-console.json | 215 +++++++++++------- app/config/specs/swagger2-latest-server.json | 180 ++++++++------- 6 files changed, 460 insertions(+), 344 deletions(-) diff --git a/app/config/specs/open-api3-latest-client.json b/app/config/specs/open-api3-latest-client.json index 820b1f55e0..316fe13116 100644 --- a/app/config/specs/open-api3-latest-client.json +++ b/app/config/specs/open-api3-latest-client.json @@ -3383,7 +3383,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3404,7 +3404,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3423,7 +3424,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -4365,7 +4367,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 7f57dfc437..5b1d6f0d1e 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -3387,7 +3387,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3408,7 +3408,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3427,7 +3428,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -7823,7 +7825,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -11585,7 +11587,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -11692,7 +11694,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -11718,54 +11720,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/healthStatus" - } - } - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -11788,7 +11742,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -11849,7 +11803,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -11910,7 +11864,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -11982,7 +11936,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -12081,8 +12035,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -12130,7 +12085,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -12191,7 +12146,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -12252,7 +12207,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -12313,7 +12268,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -12374,7 +12329,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -12413,14 +12368,14 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -12434,13 +12389,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12474,10 +12429,71 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/healthQueue" + } + } + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "schema": { + "type": "integer", + "format": "int32", + "default": 5000 + }, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "tags": [ "health" ], @@ -12495,13 +12511,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12557,7 +12573,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -12714,7 +12730,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, @@ -17609,17 +17625,17 @@ }, "endpoint": { "type": "string", - "description": "Source's Appwrite Endpoint", + "description": "Source Appwrite endpoint", "x-example": "https:\/\/example.com" }, "projectId": { "type": "string", - "description": "Source's Project ID", + "description": "Source Project ID", "x-example": "" }, "apiKey": { "type": "string", - "description": "Source's API Key", + "description": "Source API Key", "x-example": "" } }, @@ -36052,6 +36068,20 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "Aggregated number of files transformations per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of files transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -36059,7 +36089,9 @@ "filesTotal", "filesStorageTotal", "files", - "storage" + "storage", + "imageTransformations", + "imageTransformationsTotal" ] }, "usageFunctions": { @@ -36597,6 +36629,14 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ @@ -36628,7 +36668,8 @@ "authPhoneEstimate", "authPhoneCountryBreakdown", "databasesReads", - "databasesWrites" + "databasesWrites", + "imageTransformations" ] }, "headers": { diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 68d408762a..280c080514 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -3077,7 +3077,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3098,7 +3098,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3117,7 +3118,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -7381,7 +7383,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -10461,7 +10463,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -10570,7 +10572,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -10597,55 +10599,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/healthStatus" - } - } - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [], - "Key": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -10668,7 +10621,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -10730,7 +10683,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -10792,7 +10745,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -10865,7 +10818,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -10966,8 +10919,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -11015,7 +10969,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -11077,7 +11031,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -11139,7 +11093,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -11201,7 +11155,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -11263,7 +11217,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -11303,14 +11257,14 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -11324,13 +11278,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11365,10 +11319,72 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/healthQueue" + } + } + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [], + "Key": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "schema": { + "type": "integer", + "format": "int32", + "default": 5000 + }, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "tags": [ "health" ], @@ -11386,13 +11402,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11449,7 +11465,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -11609,7 +11625,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/swagger2-latest-client.json b/app/config/specs/swagger2-latest-client.json index c0980c44ce..8960bfaa5c 100644 --- a/app/config/specs/swagger2-latest-client.json +++ b/app/config/specs/swagger2-latest-client.json @@ -3557,7 +3557,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3577,7 +3577,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3596,7 +3597,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -4547,7 +4549,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 94b0d55199..f13db98150 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -3577,7 +3577,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3597,7 +3597,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3616,7 +3617,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -8012,7 +8014,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -11810,7 +11812,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -11919,7 +11921,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -11945,56 +11947,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "schema": { - "$ref": "#\/definitions\/healthStatus" - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -12019,7 +11971,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -12080,7 +12032,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -12141,7 +12093,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -12211,7 +12163,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -12309,8 +12261,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -12357,7 +12310,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -12418,7 +12371,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -12479,7 +12432,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -12540,7 +12493,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -12601,7 +12554,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -12638,10 +12591,10 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "consumes": [ "application\/json" ], @@ -12651,7 +12604,7 @@ "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -12661,13 +12614,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12699,10 +12652,71 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "consumes": [ + "application\/json" + ], + "produces": [ + "application\/json" + ], + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "schema": { + "$ref": "#\/definitions\/healthQueue" + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "type": "integer", + "format": "int32", + "default": 5000, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "consumes": [ "application\/json" ], @@ -12722,13 +12736,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12784,7 +12798,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -12945,7 +12959,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, @@ -18070,19 +18084,19 @@ }, "endpoint": { "type": "string", - "description": "Source's Appwrite Endpoint", + "description": "Source Appwrite endpoint", "default": null, "x-example": "https:\/\/example.com" }, "projectId": { "type": "string", - "description": "Source's Project ID", + "description": "Source Project ID", "default": null, "x-example": "" }, "apiKey": { "type": "string", - "description": "Source's API Key", + "description": "Source API Key", "default": null, "x-example": "" } @@ -36608,6 +36622,21 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "Aggregated number of files transformations per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of files transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -36615,7 +36644,9 @@ "filesTotal", "filesStorageTotal", "files", - "storage" + "storage", + "imageTransformations", + "imageTransformationsTotal" ] }, "usageFunctions": { @@ -37185,6 +37216,15 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ @@ -37216,7 +37256,8 @@ "authPhoneEstimate", "authPhoneCountryBreakdown", "databasesReads", - "databasesWrites" + "databasesWrites", + "imageTransformations" ] }, "headers": { diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index e38495629c..749f87553b 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -3261,7 +3261,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3281,7 +3281,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3300,7 +3301,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -7552,7 +7554,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -10689,7 +10691,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -10800,7 +10802,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -10827,57 +10829,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "schema": { - "$ref": "#\/definitions\/healthStatus" - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [], - "Key": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -10902,7 +10853,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -10964,7 +10915,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -11026,7 +10977,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -11097,7 +11048,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -11197,8 +11148,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -11245,7 +11197,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -11307,7 +11259,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -11369,7 +11321,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -11431,7 +11383,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -11493,7 +11445,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -11531,10 +11483,10 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "consumes": [ "application\/json" ], @@ -11544,7 +11496,7 @@ "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -11554,13 +11506,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11593,10 +11545,72 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "consumes": [ + "application\/json" + ], + "produces": [ + "application\/json" + ], + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "schema": { + "$ref": "#\/definitions\/healthQueue" + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [], + "Key": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "type": "integer", + "format": "int32", + "default": 5000, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "consumes": [ "application\/json" ], @@ -11616,13 +11630,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11679,7 +11693,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -11843,7 +11857,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, From df3746271684ad08c1b1379bf9d784122bdd5262 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 13:20:55 +0000 Subject: [PATCH 36/65] chore: fix typing and tests --- app/config/specs/open-api3-latest-console.json | 10 ++++------ app/config/specs/swagger2-latest-console.json | 11 ++++------- src/Appwrite/Utopia/Response/Model/UsageProject.php | 9 ++++----- tests/e2e/General/UsageTest.php | 2 +- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 5b1d6f0d1e..5b31a62c23 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -36631,12 +36631,10 @@ "x-example": [] }, "imageTransformations": { - "type": "array", - "description": "An array of aggregated number of image transformations.", - "items": { - "$ref": "#\/components\/schemas\/metric" - }, - "x-example": [] + "type": "integer", + "description": "Total aggregated number of image transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index f13db98150..5c2dabd318 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -37218,13 +37218,10 @@ "x-example": [] }, "imageTransformations": { - "type": "array", - "description": "An array of aggregated number of image transformations.", - "items": { - "type": "object", - "$ref": "#\/definitions\/metric" - }, - "x-example": [] + "type": "integer", + "description": "Total aggregated number of image transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 6f74ec561b..bdb7d6a897 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -198,11 +198,10 @@ class UsageProject extends Model 'array' => true ]) ->addRule('imageTransformations', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'An array of aggregated number of image transformations.', - 'default' => [], - 'example' => [], - 'array' => true + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of image transformations.', + 'default' => 0, + 'example' => 0, ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c0d0c80eb1..9061f95a96 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -150,7 +150,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(29, count($response['body'])); + $this->assertEquals(30, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); From 02de0d2432750728768fe6efb0a81cd21eff54d9 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 13:29:32 +0000 Subject: [PATCH 37/65] chore: fix stats count --- tests/e2e/General/UsageTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 9061f95a96..2e13dcdb98 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -331,7 +331,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(29, count($response['body'])); + $this->assertEquals(30, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -552,7 +552,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(29, count($response['body'])); + $this->assertEquals(30, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); From 712c33064a3bd4d7af83278dab9ad13a17ff8759 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 1 Mar 2025 17:18:44 +0000 Subject: [PATCH 38/65] chore: fix sending of transformation stats --- app/controllers/api/project.php | 4 +++- src/Appwrite/Utopia/Response/Model/UsageProject.php | 6 ++++++ tests/e2e/General/UsageTest.php | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index cf2a128939..986264e3c5 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -76,6 +76,7 @@ App::get('/v1/project/usage') METRIC_BUILDS_MB_SECONDS, METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, + METRIC_FILES_IMAGES_TRANSFORMED, ] ]; @@ -364,7 +365,8 @@ App::get('/v1/project/usage') 'authPhoneTotal' => $authPhoneTotal, 'authPhoneEstimate' => $authPhoneEstimate, 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, - 'imageTransformations' => $total[METRIC_FILES_IMAGES_TRANSFORMED], + 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], + 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index bdb7d6a897..6ac8830ac5 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -197,6 +197,12 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('imageTransformationsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'An array of aggregated number of image transformations.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('imageTransformations', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of image transformations.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 2e13dcdb98..33a1408704 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -150,7 +150,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(30, count($response['body'])); + $this->assertEquals(31, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); @@ -331,7 +331,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(30, count($response['body'])); + $this->assertEquals(31, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -552,7 +552,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(30, count($response['body'])); + $this->assertEquals(31, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); From aaee99e08b950affc760ee193776e0ec4d8df767 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 1 Mar 2025 17:19:55 +0000 Subject: [PATCH 39/65] chore: generate specs --- app/config/specs/open-api3-latest-console.json | 7 +++++++ app/config/specs/swagger2-latest-console.json | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 5b31a62c23..bc216226fb 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -36630,6 +36630,12 @@ }, "x-example": [] }, + "imageTransformationsTotal": { + "type": "integer", + "description": "An array of aggregated number of image transformations.", + "x-example": 0, + "format": "int32" + }, "imageTransformations": { "type": "integer", "description": "Total aggregated number of image transformations.", @@ -36667,6 +36673,7 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", + "imageTransformationsTotal", "imageTransformations" ] }, diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 5c2dabd318..2dddc72802 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -37217,6 +37217,12 @@ }, "x-example": [] }, + "imageTransformationsTotal": { + "type": "integer", + "description": "An array of aggregated number of image transformations.", + "x-example": 0, + "format": "int32" + }, "imageTransformations": { "type": "integer", "description": "Total aggregated number of image transformations.", @@ -37254,6 +37260,7 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", + "imageTransformationsTotal", "imageTransformations" ] }, From 6900b717da261b71bfbb928c96e8fa264f6d5ea4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 1 Mar 2025 17:45:20 +0000 Subject: [PATCH 40/65] chore: updated to use logsdb --- app/controllers/api/project.php | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 986264e3c5..b267e8e51e 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -40,14 +40,18 @@ App::get('/v1/project/usage') ->param('endDate', '', new DateTimeValidator(), 'End date for the usage') ->param('period', '1d', new WhiteList(['1h', '1d']), 'Period used', true) ->inject('response') + ->inject('project') ->inject('dbForProject') + ->inject('getLogsDB') ->inject('smsRates') - ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject, array $smsRates) { + ->action(function (string $startDate, string $endDate, string $period, Response $response, Document $project, Database $dbForProject, callable $getLogsDB, array $smsRates) { $stats = $total = $usage = []; $format = 'Y-m-d 00:00:00'; $firstDay = (new DateTime($startDate))->format($format); $lastDay = (new DateTime($endDate))->format($format); + $dbForLogs = call_user_func($getLogsDB, $project); + $metrics = [ 'total' => [ METRIC_EXECUTIONS, @@ -95,9 +99,11 @@ App::get('/v1/project/usage') '1d' => 'Y-m-d\T00:00:00.000P', }; - Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, $limit, &$total, &$stats) { + Authorization::skip(function () use ($dbForProject, $dbForLogs, $firstDay, $lastDay, $period, $metrics, $limit, &$total, &$stats) { foreach ($metrics['total'] as $metric) { - $result = $dbForProject->findOne('stats', [ + $db = ($metric === METRIC_FILES_IMAGES_TRANSFORMED) ? $dbForLogs : $dbForProject; + + $result = $db->findOne('stats', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -105,7 +111,9 @@ App::get('/v1/project/usage') } foreach ($metrics['period'] as $metric) { - $results = $dbForProject->find('stats', [ + $db = ($metric === METRIC_FILES_IMAGES_TRANSFORMED) ? $dbForLogs : $dbForProject; + + $results = $db->find('stats', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::greaterThanEqual('time', $firstDay), From 5becd0a5dc9553fb0e120ad5ee52e0785f55a28f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 3 Mar 2025 16:04:31 +0545 Subject: [PATCH 41/65] Revert "Fix: stats resources only queue projects accessed in last 3 hours" --- src/Appwrite/Platform/Tasks/StatsResources.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index c318f92cf1..ac3b9ead73 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -62,7 +62,7 @@ class StatsResources extends Action Authorization::disable(); Authorization::setDefaultStatus(false); - $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('3 hours')); + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); /** * For each project that were accessed in last 24 hours */ From b314ae82084fea574ac63d0a9f18bb2990ef88b1 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 3 Mar 2025 15:19:49 +0200 Subject: [PATCH 42/65] disable transformedAt update for console users --- app/controllers/api/storage.php | 20 ++++++++------------ app/controllers/shared/api.php | 14 ++++++++------ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f180c22acf..5bf52f995f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -6,7 +6,6 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; @@ -942,8 +941,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->inject('queueForStatsUsage') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1071,15 +1069,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - $queueForStatsUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - - $transformedAt = $file->getAttribute('transformedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { - $file->setAttribute('transformedAt', DateTime::now()); - Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + //Do not update transformedAt if it's a console user + if (!Auth::isPrivilegedUser(Authorization::getRoles())) { + $transformedAt = $file->getAttribute('transformedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { + $file->setAttribute('transformedAt', DateTime::now()); + Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + } } $response diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 7f7b73ab0c..1015400a12 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -534,7 +534,7 @@ App::init() if ($type === 'bucket') { $bucketId = $parts[1] ?? null; - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAppUser && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); @@ -560,11 +560,13 @@ App::init() if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - - $transformedAt = $file->getAttribute('transformedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { - $file->setAttribute('transformedAt', DateTime::now()); - Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + //Do not update transformedAt if it's a console user + if (!Auth::isPrivilegedUser(Authorization::getRoles())) { + $transformedAt = $file->getAttribute('transformedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { + $file->setAttribute('transformedAt', DateTime::now()); + Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + } } } From c6b8e649fef5108bcbe70bc54270b01d851cd09b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 14:34:10 +0000 Subject: [PATCH 43/65] chore: added maxtimeout to testChannelExecutions --- tests/e2e/Services/Realtime/RealtimeCustomClientTest.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index dda524fc7c..c72dbf85ec 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -1311,6 +1311,8 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals($deployment['headers']['status-code'], 202); $this->assertNotEmpty($deployment['body']['$id']); + $maxTimeSeconds = 10; + $startTime = time(); // Poll until deployment is built while (true) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ @@ -1326,6 +1328,10 @@ class RealtimeCustomClientTest extends Scope break; } + if (time() - $startTime >= $maxTimeSeconds) { + throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); + } + \sleep(1); } From e865ca74d775393d12af377b4f120fd087a4d32b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 14:41:11 +0000 Subject: [PATCH 44/65] chore: added timeout to webhooks base test --- tests/e2e/Services/Webhooks/WebhooksBase.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php index 2ef41003ee..ffcb1ce980 100644 --- a/tests/e2e/Services/Webhooks/WebhooksBase.php +++ b/tests/e2e/Services/Webhooks/WebhooksBase.php @@ -14,6 +14,8 @@ trait WebhooksBase { protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void { + $maxTimeSeconds = 10; + $startTime = time(); while (true) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', @@ -28,6 +30,10 @@ trait WebhooksBase break; } + if (time() - $startTime >= $maxTimeSeconds) { + throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); + } + \sleep(1); } From fc05a4f1e30e164c54e1330ef2cdcb77cd3cc36b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 16:07:33 +0000 Subject: [PATCH 45/65] chore: updated to use assertEventually --- .../Realtime/RealtimeCustomClientTest.php | 19 ++----------- tests/e2e/Services/Webhooks/WebhooksBase.php | 28 ++++++------------- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index c72dbf85ec..e356397408 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -1311,29 +1311,16 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals($deployment['headers']['status-code'], 202); $this->assertNotEmpty($deployment['body']['$id']); - $maxTimeSeconds = 10; - $startTime = time(); // Poll until deployment is built - while (true) { + $this->assertEventually(function () use ($function, $deploymentId) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], ]); - if ( - $deployment['headers']['status-code'] >= 400 - || \in_array($deployment['body']['status'], ['ready', 'failed']) - ) { - break; - } - - if (time() - $startTime >= $maxTimeSeconds) { - throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); - } - - \sleep(1); - } + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); + }); $response = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([ 'content-type' => 'application/json', diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php index ffcb1ce980..b7985b5be1 100644 --- a/tests/e2e/Services/Webhooks/WebhooksBase.php +++ b/tests/e2e/Services/Webhooks/WebhooksBase.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Services\Webhooks; +use Appwrite\Tests\Async; use Appwrite\Tests\Retry; use CURLFile; use Tests\E2E\Client; @@ -12,35 +13,22 @@ use Utopia\Database\Validator\Datetime as DatetimeValidator; trait WebhooksBase { + use Async; + protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void { - $maxTimeSeconds = 10; - $startTime = time(); - while (true) { + $this->assertEventually(function () use ($functionId, $deploymentId, $checkForSuccess) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], ]); - if ( - $deployment['headers']['status-code'] >= 400 - || \in_array($deployment['body']['status'], ['ready', 'failed']) - ) { - break; + if ($checkForSuccess) { + $this->assertEquals(200, $deployment['headers']['status-code']); + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); } - - if (time() - $startTime >= $maxTimeSeconds) { - throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); - } - - \sleep(1); - } - - if ($checkForSuccess) { - $this->assertEquals(200, $deployment['headers']['status-code']); - $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); - } + }); } public static function getWebhookSignature(array $webhook, string $signatureKey): string From fe7aeefabf9f2f1523df07ebfa3a060099688d70 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 16:20:32 +0000 Subject: [PATCH 46/65] chore: remove redundant param --- tests/e2e/Services/Webhooks/WebhooksBase.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php index b7985b5be1..c743810feb 100644 --- a/tests/e2e/Services/Webhooks/WebhooksBase.php +++ b/tests/e2e/Services/Webhooks/WebhooksBase.php @@ -15,19 +15,17 @@ trait WebhooksBase { use Async; - protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void + protected function awaitDeploymentIsBuilt($functionId, $deploymentId): void { - $this->assertEventually(function () use ($functionId, $deploymentId, $checkForSuccess) { + $this->assertEventually(function () use ($functionId, $deploymentId) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], ]); - if ($checkForSuccess) { - $this->assertEquals(200, $deployment['headers']['status-code']); - $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); - } + $this->assertEquals(200, $deployment['headers']['status-code']); + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); }); } From 9a6de87f7dcbdeff0d6c70f18cb1923ed87c2a6b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 16:31:41 +0000 Subject: [PATCH 47/65] chore: usagetest and migrationsbase --- tests/e2e/General/UsageTest.php | 67 ++++++++----------- .../Services/Migrations/MigrationsBase.php | 23 +++---- 2 files changed, 37 insertions(+), 53 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index df780d8f47..27d521aa6a 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1142,49 +1142,38 @@ class UsageTest extends Scope $tries = 0; - while (true) { - try { - // Compare new values with old values - $response = $this->client->call( - Client::METHOD_GET, - '/functions/' . $functionId . '/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () use ($functionId, $functionsMetrics, $projectMetrics) { + // Compare new values with old values + $response = $this->client->call( + Client::METHOD_GET, + '/functions/' . $functionId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(19, count($response['body'])); - $this->assertEquals('30d', $response['body']['range']); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(19, count($response['body'])); + $this->assertEquals('30d', $response['body']['range']); - // Check if the new values are greater than the old values - $this->assertEquals($functionsMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); - $this->assertGreaterThan($functionsMetrics['executionsTimeTotal'], $response['body']['executionsTimeTotal']); - $this->assertGreaterThan($functionsMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); + // Check if the new values are greater than the old values + $this->assertEquals($functionsMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); + $this->assertGreaterThan($functionsMetrics['executionsTimeTotal'], $response['body']['executionsTimeTotal']); + $this->assertGreaterThan($functionsMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals($projectMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); - $this->assertGreaterThan($projectMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); - - break; - } catch (ExpectationFailedException $th) { - if ($tries >= 5) { - throw $th; - } else { - $tries++; - sleep(5); - } - } - } + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($projectMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); + $this->assertGreaterThan($projectMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); + }); } public function tearDown(): void diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 381706f5ee..6c468ee730 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -53,8 +53,9 @@ trait MigrationsBase $this->assertNotEmpty($migration['body']); $this->assertNotEmpty($migration['body']['$id']); - $attempts = 0; - while ($attempts < 5) { + $migrationResult = []; + + $this->assertEventually(function () use ($migration, &$migrationResult) { $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migration['body']['$id'], [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getDestinationProject()['$id'], @@ -66,24 +67,18 @@ trait MigrationsBase $this->assertNotEmpty($response['body']['$id']); if ($response['body']['status'] === 'failed') { - $this->fail('Migration failed', json_encode($response['body'], JSON_PRETTY_PRINT)); + $this->fail('Migration failed' . json_encode($response['body'], JSON_PRETTY_PRINT)); } $this->assertNotEquals('failed', $response['body']['status']); + $this->assertEquals('completed', $response['body']['status']); - if ($response['body']['status'] === 'completed') { - return $response['body']; - } + $migrationResult = $response['body']; - if ($attempts === 4) { - $this->assertEquals('completed', $response['body']['status']); - } + return true; + }); - $attempts++; - sleep(5); - } - - return []; + return $migrationResult; } /** From 5d6154444462ca8b49f0c8b1f97ec3e15e1a1e1b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 05:31:35 +0000 Subject: [PATCH 48/65] Fix: use receivedAt date when available --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 5d7240f4b5..292fdffc00 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -142,7 +142,11 @@ class StatsUsageDump extends Action } foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); + $time = null; + + if ($period !== 'inf') { + $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + } $id = \md5("{$time}_{$period}_{$key}"); $document = new Document([ From dd5dd4cc43c26404a8e73a7e78e7063bac3464c2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 05:34:02 +0000 Subject: [PATCH 49/65] Same for database metrics --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 292fdffc00..294269d586 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -134,7 +134,7 @@ class StatsUsageDump extends Action if (str_contains($key, METRIC_DATABASES_STORAGE)) { try { - $this->handleDatabaseStorage($key, $dbForProject, $project); + $this->handleDatabaseStorage($key, $dbForProject, $project, $receivedAt); } catch (\Exception $e) { console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); } @@ -175,12 +175,12 @@ class StatsUsageDump extends Action } } - private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project): void + private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project, string $receivedAt): void { $data = explode('.', $key); $start = microtime(true); - $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) { + $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) use ($receivedAt) { $id = \md5("{$time}_{$period}_{$key}"); $document = new Document([ @@ -201,7 +201,11 @@ class StatsUsageDump extends Action }; foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); + $time = null; + + if ($period !== 'inf') { + $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + } $id = \md5("{$time}_{$period}_{$key}"); $value = 0; From 4d5d30ba7bbab63508467788ecf5fd7a04df222e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 05:50:21 +0000 Subject: [PATCH 50/65] Fix default value --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 294269d586..e03bb86b58 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -117,7 +117,7 @@ class StatsUsageDump extends Action $project = new Document($stats['project'] ?? []); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? 'NONE'; + $receivedAt = $stats['receivedAt'] ?? null; if ($numberOfKeys === 0) { continue; } @@ -145,7 +145,7 @@ class StatsUsageDump extends Action $time = null; if ($period !== 'inf') { - $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); } $id = \md5("{$time}_{$period}_{$key}"); @@ -204,7 +204,7 @@ class StatsUsageDump extends Action $time = null; if ($period !== 'inf') { - $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); } $id = \md5("{$time}_{$period}_{$key}"); From 0659dce9357584dcf3e90283b739303a2fa6fdee Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 06:22:58 +0000 Subject: [PATCH 51/65] Fix: disable dual writing --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index e03bb86b58..81acd4e4b0 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -328,7 +328,7 @@ class StatsUsageDump extends Action protected function writeToLogsDB(Document $project, Document $document): void { - if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { Console::log('Dual Writing is disabled. Skipping...'); return; } From 8b4bc7e4a6b1a39af6b171bc3362bf7b5e1cafca Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 18:19:23 +0200 Subject: [PATCH 52/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 977aff3cfb..2d48750e9d 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,15 +17,15 @@ use Utopia\System\System; class Audits extends Action { - private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - private const BATCH_SIZE_PRODUCTION = 5_000; - private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + protected const BATCH_SIZE_PRODUCTION = 5_000; + protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds - private int $lastTriggeredTime = 0; + protected int $lastTriggeredTime = 0; - private array $logs = []; + protected array $logs = []; - private function getBatchSize(): int + protected function getBatchSize(): int { return System::getEnv('_APP_ENV', 'development') === 'development' ? self::BATCH_SIZE_DEVELOPMENT From f761829d6053580defc8eada81cb72061c42b07f Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 18:49:31 +0200 Subject: [PATCH 53/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 2d48750e9d..2a43a9e24f 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,7 +17,7 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 2; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds From 76dc8c29d5e5bf8c2f8149e4a186331753082acc Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:07:45 +0200 Subject: [PATCH 54/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 2a43a9e24f..f22df17d31 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -21,7 +21,7 @@ class Audits extends Action protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds - protected int $lastTriggeredTime = 0; + private int $lastTriggeredTime = 0; protected array $logs = []; From ce054e7088db0966c3cbf0d7527b284b273af3c6 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:12:20 +0200 Subject: [PATCH 55/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index f22df17d31..85fe18ce9e 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -23,7 +23,7 @@ class Audits extends Action private int $lastTriggeredTime = 0; - protected array $logs = []; + private array $logs = []; protected function getBatchSize(): int { From 09aa383cf338fe53053725c1ade509a249ed9f32 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:15:00 +0200 Subject: [PATCH 56/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 85fe18ce9e..3221010a07 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -25,6 +25,7 @@ class Audits extends Action private array $logs = []; + protected function getBatchSize(): int { return System::getEnv('_APP_ENV', 'development') === 'development' From 1838b9c4e879f2eca2fd775dcd32df9821e35db5 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:49:10 +0200 Subject: [PATCH 57/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 3221010a07..c663ffc6d4 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,9 +17,9 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 2; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 26; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + protected const BATCH_AGGREGATION_INTERVAL = 300; // in seconds private int $lastTriggeredTime = 0; From c443c8fdab744b6c532196145ee046ac98612b1c Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:49:37 +0200 Subject: [PATCH 58/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index c663ffc6d4..3367bc1268 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,9 +17,9 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 26; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 6; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 300; // in seconds + protected const BATCH_AGGREGATION_INTERVAL = 120; // in seconds private int $lastTriggeredTime = 0; From f2ad3a501ed4a538f655592df4aa0b31e3105399 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 20:34:56 +0200 Subject: [PATCH 59/65] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 3367bc1268..269b29c4c8 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,9 +17,9 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 6; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 120; // in seconds + protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; From 1d2883f47250b45837d87905c8ffcf0d4b480f5f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 04:46:06 +0000 Subject: [PATCH 60/65] chore: fix model for image transformations in usage project --- src/Appwrite/Utopia/Response/Model/UsageProject.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 6ac8830ac5..6737cbf6ef 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -197,18 +197,19 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('imageTransformations', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of image transformations.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('imageTransformationsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'An array of aggregated number of image transformations.', 'default' => 0, 'example' => 0, ]) - ->addRule('imageTransformations', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of image transformations.', - 'default' => 0, - 'example' => 0, - ]) ; } From 2c6f731a3820333a18fbcafbd43f478349f0a0bd Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 04:49:34 +0000 Subject: [PATCH 61/65] fix: description --- src/Appwrite/Utopia/Response/Model/UsageProject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 6737cbf6ef..395b19b7cd 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -206,7 +206,7 @@ class UsageProject extends Model ]) ->addRule('imageTransformationsTotal', [ 'type' => self::TYPE_INTEGER, - 'description' => 'An array of aggregated number of image transformations.', + 'description' => 'Total aggregated number of image transformations.', 'default' => 0, 'example' => 0, ]) From 6024d01e7ba7413ce462f09064b9a2abfd502a88 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Mar 2025 05:25:50 +0000 Subject: [PATCH 62/65] Feat: calculate database storage in stats-resources - Calculate database storage in stats-resources worker --- .../Platform/Workers/StatsResources.php | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 62046bd186..969d43e895 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -182,7 +182,7 @@ class StatsResources extends Action } try { - $this->countForDatabase($dbForProject, $dbForLogs, $region); + $this->countForDatabase($dbForProject, $region); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]); } @@ -242,42 +242,54 @@ class StatsResources extends Action $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations); } - protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region) + protected function countForDatabase(Database $dbForProject, string $region) { $totalCollections = 0; $totalDocuments = 0; - $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) { + $totalDatabaseStorage = 0; + + $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage) { $collections = $dbForProject->count('database_' . $database->getInternalId()); $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS); $this->createStatsDocuments($region, $metric, $collections); - $documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region); + [$documents, $storage] = $this->countForCollections($dbForProject, $database, $region); + $totalDatabaseStorage += $storage; $totalDocuments += $documents; $totalCollections += $collections; }); $this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections); $this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments); + $this->createStatsDocuments($region, METRIC_DATABASES_STORAGE, $totalDatabaseStorage); } - protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int + protected function countForCollections(Database $dbForProject, Document $database, string $region): array { $databaseDocuments = 0; - $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) { + $databaseStorage = 0; + $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $database, $region, &$databaseStorage, &$databaseDocuments) { $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); $this->createStatsDocuments($region, $metric, $documents); - $databaseDocuments += $documents; + + $collectionStorage = $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE); + $this->createStatsDocuments($region, $metric, $collectionStorage); + $databaseStorage += $collectionStorage; + }); $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS); $this->createStatsDocuments($region, $metric, $databaseDocuments); - return $databaseDocuments; + $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE); + $this->createStatsDocuments($region, $metric, $databaseStorage); + + return [$databaseDocuments, $databaseStorage]; } protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) From 2afb8a14ac7f541eda336cd85c71e76e31e4b83d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Mar 2025 05:27:56 +0000 Subject: [PATCH 63/65] Skip in dual writing --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 81acd4e4b0..119a9e7288 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -48,6 +48,7 @@ class StatsUsageDump extends Action METRIC_BUILDS => true, METRIC_COLLECTIONS => true, METRIC_DOCUMENTS => true, + METRIC_DATABASES_STORAGE => true, ]; /** @@ -63,6 +64,7 @@ class StatsUsageDump extends Action '.deployments.storage', '.builds', '.builds.storage', + '.databases.storage' ]; /** From 0ce4555f70312b0d9f1de93d91f96f2ed70d3192 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 17:42:43 +0000 Subject: [PATCH 64/65] chore: added auth group to create phone token --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index a634618e6e..20f64496ac 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2400,7 +2400,7 @@ App::put('/v1/account/sessions/phone') App::post('/v1/account/tokens/phone') ->alias('/v1/account/sessions/phone') ->desc('Create phone token') - ->groups(['api', 'account']) + ->groups(['api', 'account', 'auth']) ->label('scope', 'sessions.write') ->label('auth.type', 'phone') ->label('audits.event', 'session.create') From 36b047529843dc23f3b94d2b23cb2fb6c2f0b36b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 18:40:41 +0000 Subject: [PATCH 65/65] chore: update tests --- .../Account/AccountCustomClientTest.php | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 439fa24fb6..daa5bcbff8 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2419,6 +2419,33 @@ class AccountCustomClientTest extends Scope $message = $smsRequest['data']['message']; $token = substr($message, 0, 6); + /** + * Test for FAILURE + */ + + // disable phone sessions + $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ]), [ + 'status' => false, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(false, $response['body']['authPhone']); + + $response = $this->client->call(Client::METHOD_POST, '/account/verification/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + + $this->assertEquals(501, $response['headers']['status-code']); + $this->assertEquals("Phone authentication is disabled for this project", $response['body']['message']); + return \array_merge($data, [ 'token' => \substr($smsRequest['data']['message'], 0, 6) ]);