diff --git a/composer.json b/composer.json index b8c5afa109..176b5889fc 100644 --- a/composer.json +++ b/composer.json @@ -55,7 +55,7 @@ "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", - "utopia-php/fetch": "0.3.*", + "utopia-php/fetch": "0.4.*", "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", diff --git a/composer.lock b/composer.lock index bd9366a34c..3772b25707 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5b3c46863e4571c838c30090ad96436c", + "content-hash": "c99b4733669c17013e211c7dc54a86f6", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.63.0", + "version": "0.63.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a" + "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/51ff0a6514e6eda44a6da016d838e7e55282e01a", - "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/ad191bf34151815f716f553796a363ff2b6ef7d3", + "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.0" + "source": "https://github.com/utopia-php/database/tree/0.63.1" }, - "time": "2025-03-26T10:35:37+00:00" + "time": "2025-03-27T04:58:07+00:00" }, { "name": "utopia-php/domains", @@ -3659,16 +3659,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.3.1", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0" + "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", - "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/46e791ff6a95864517750b9df6bbf4a17e3c9c4e", + "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e", "shasum": "" }, "require": { @@ -3692,9 +3692,9 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.3.1" + "source": "https://github.com/utopia-php/fetch/tree/0.4.0" }, - "time": "2025-03-05T18:08:55+00:00" + "time": "2025-03-11T21:06:56+00:00" }, { "name": "utopia-php/framework", @@ -3950,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.3", + "version": "0.8.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/845fd04ccf5e0edb03c184b864e0596080a432b8", + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8", "shasum": "" }, "require": { @@ -3967,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.63.*", + "utopia-php/database": "0.*.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.3" + "source": "https://github.com/utopia-php/migration/tree/0.8.4" }, - "time": "2025-03-26T10:45:51+00:00" + "time": "2025-03-28T02:08:22+00:00" }, { "name": "utopia-php/orchestration", @@ -4211,23 +4211,23 @@ }, { "name": "utopia-php/queue", - "version": "0.9.0", + "version": "0.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7" + "reference": "32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/077075f1d57afa430f76c35ed3bf4616e0eee8e7", - "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32", + "reference": "32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32", "shasum": "" }, "require": { "php": ">=8.3", "php-amqplib/php-amqplib": "^3.7", "utopia-php/cli": "0.15.*", - "utopia-php/fetch": "^0.3.0", + "utopia-php/fetch": "0.4.*", "utopia-php/framework": "0.33.*", "utopia-php/telemetry": "0.1.*" }, @@ -4270,9 +4270,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.9.0" + "source": "https://github.com/utopia-php/queue/tree/0.9.1" }, - "time": "2025-03-13T12:22:41+00:00" + "time": "2025-03-28T19:49:36+00:00" }, { "name": "utopia-php/registry", @@ -4775,16 +4775,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.10", + "version": "0.40.11", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051" + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", "shasum": "" }, "require": { @@ -4820,9 +4820,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" }, - "time": "2025-03-25T13:44:16+00:00" + "time": "2025-03-26T10:53:16+00:00" }, { "name": "doctrine/annotations", @@ -7154,16 +7154,16 @@ }, { "name": "symfony/console", - "version": "v7.2.1", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" + "reference": "e51498ea18570c062e7df29d05a7003585b19b88" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "url": "https://api.github.com/repos/symfony/console/zipball/e51498ea18570c062e7df29d05a7003585b19b88", + "reference": "e51498ea18570c062e7df29d05a7003585b19b88", "shasum": "" }, "require": { @@ -7227,7 +7227,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.1" + "source": "https://github.com/symfony/console/tree/v7.2.5" }, "funding": [ { @@ -7243,7 +7243,7 @@ "type": "tidelift" } ], - "time": "2024-12-11T03:49:26+00:00" + "time": "2025-03-12T08:11:12+00:00" }, { "name": "symfony/filesystem", @@ -7758,16 +7758,16 @@ }, { "name": "symfony/process", - "version": "v7.2.4", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf" + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", + "url": "https://api.github.com/repos/symfony/process/zipball/87b7c93e57df9d8e39a093d32587702380ff045d", + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d", "shasum": "" }, "require": { @@ -7799,7 +7799,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.2.4" + "source": "https://github.com/symfony/process/tree/v7.2.5" }, "funding": [ { @@ -7815,7 +7815,7 @@ "type": "tidelift" } ], - "time": "2025-02-05T08:33:46+00:00" + "time": "2025-03-13T12:21:46+00:00" }, { "name": "symfony/string", diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 962bc622d1..ed5ff8010a 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,12 +12,12 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; -use Utopia\Queue\Result\Commit; -use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { + protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,7 +27,9 @@ class Audits extends Action protected function getBatchSize(): int { - return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; } public static function getName(): string @@ -55,13 +57,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return Commit|NoCommit + * @return void * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit + public function action(Message $message, callable $getProjectDB, Document $project): void { $payload = $message->getPayload() ?? []; @@ -117,34 +119,29 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs), 0); - $shouldProcessBatch = $logCount >= $batchSize; - if (!$shouldProcessBatch && $logCount > 0) { + $shouldProcessBatch = \count($this->logs) >= $batchSize; + if (!$shouldProcessBatch && \count($this->logs) > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if (!$shouldProcessBatch) { - return new NoCommit(); - } + if ($shouldProcessBatch) { + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); - - unset($this->logs[$internalId]); + unset($this->logs[$internalId]); + } + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + $this->lastTriggeredTime = time(); } - return new Commit(); - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - return new NoCommit(); - } finally { - $this->lastTriggeredTime = time(); } } } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 5033822241..a9b83976a4 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -405,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select($this->selects), + Query::select([...$this->selects, 'accessedAt']), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,9 +439,11 @@ class Deletes extends Action /** @var Database $dbForProject*/ $dbForProject = $getProjectDB($project); + $selects = [...$this->selects, 'time']; + // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select($this->selects), + Query::select($selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -454,7 +456,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select($this->selects), + Query::select($selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -751,7 +753,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select($this->selects), + Query::select([...$this->selects, '$createdAt']), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -772,7 +774,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select($this->selects), + Query::select([...$this->selects, '$createdAt']), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -809,7 +811,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select($this->selects), + Query::select([...$this->selects, 'time']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(),