diff --git a/composer.json b/composer.json index b7c3711410..58b74c3fe3 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.49.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.49.*", + "utopia-php/audit": "0.50.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index 2fcea6b3e5..fd89ba3ae8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4a54d0bd5973ed68082970f317664df3", + "content-hash": "884381b7cc6c225f83c397eb472ddf11", "packages": [ { "name": "adhocore/jwt", @@ -3474,16 +3474,16 @@ }, { "name": "utopia-php/audit", - "version": "0.49.0", + "version": "0.50.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d" + "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/c0da7dcdd35fc7d3f9640ba21cc82607cf7da729", + "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729", "shasum": "" }, "require": { @@ -3515,9 +3515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.49.0" + "source": "https://github.com/utopia-php/audit/tree/0.50.0" }, - "time": "2025-02-04T07:27:18+00:00" + "time": "2025-02-12T05:30:25+00:00" }, { "name": "utopia-php/cache", diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index c0bcab1c3a..f1ae46eea7 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -6,15 +6,33 @@ use Appwrite\Auth\Auth; use Exception; use Throwable; use Utopia\Audit\Audit; +use Utopia\CLI\Console; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\System\System; class Audits extends Action { + private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + private const BATCH_SIZE_PRODUCTION = 5_000; + private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + + private int $lastTriggeredTime = 0; + + private array $logs = []; + + private function getBatchSize(): int + { + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; + } + public static function getName(): string { return 'audits'; @@ -30,6 +48,8 @@ class Audits extends Action ->inject('message') ->inject('dbForProject') ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject)); + + $this->lastTriggeredTime = time(); } @@ -44,13 +64,14 @@ class Audits extends Action */ public function action(Message $message, Database $dbForProject): void { - $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } + Console::info('Aggregating audit logs'); + $event = $payload['event'] ?? ''; $auditPayload = $payload['payload'] ?? ''; $mode = $payload['mode'] ?? ''; @@ -63,23 +84,48 @@ class Audits extends Action $userEmail = $user->getAttribute('email', ''); $userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER); - $audit = new Audit($dbForProject); - $audit->log( - userId: $user->getInternalId(), - // Pass first, most verbose event pattern - event: $event, - resource: $resource, - userAgent: $userAgent, - ip: $ip, - location: '', - data: [ + // Create event data + $eventData = [ + 'userId' => $user->getInternalId(), + 'event' => $event, + 'resource' => $resource, + 'userAgent' => $userAgent, + 'ip' => $ip, + 'location' => '', + 'data' => [ 'userId' => $user->getId(), 'userName' => $userName, 'userEmail' => $userEmail, 'userType' => $userType, 'mode' => $mode, 'data' => $auditPayload, - ] - ); + ], + 'timestamp' => DateTime::formatTz(DateTime::now()) + ]; + + $this->logs[] = $eventData; + + // Check if we should process the batch by checking both for the batch size and the elapsed time + $batchSize = $this->getBatchSize(); + $shouldProcessBatch = count($this->logs) >= $batchSize; + if (!$shouldProcessBatch && count($this->logs) > 0) { + $shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; + } + + if ($shouldProcessBatch) { + Console::log('Processing batch with ' . count($this->logs) . ' events'); + $audit = new Audit($dbForProject); + + try { + $audit->logBatch($this->logs); + Console::success('Audit logs processed successfully'); + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + // Clear the pending events after successful batch processing + $this->logs = []; + $this->lastTriggeredTime = time(); + } + } } }