From 00473d47440f6779f7579ec80ccca15941c7d584 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 10:57:43 +0530 Subject: [PATCH 1/9] feat: batch create abuse logs --- .cursorignore | 7 +++ composer.json | 2 +- composer.lock | 18 +++---- src/Appwrite/Platform/Workers/Audits.php | 60 +++++++++++++++++++----- 4 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 .cursorignore diff --git a/.cursorignore b/.cursorignore new file mode 100644 index 0000000000..95ce4247a6 --- /dev/null +++ b/.cursorignore @@ -0,0 +1,7 @@ +docs/* +public/* +vendor/* +dev/* +.vscode/* +app/assets/* +app/sdks/* \ No newline at end of file diff --git a/composer.json b/composer.json index b7c3711410..6e25f5f912 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.49.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.49.*", + "utopia-php/audit": "dev-add-batch-logging-method", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index 2fcea6b3e5..e77db78d2f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4a54d0bd5973ed68082970f317664df3", + "content-hash": "9b7ba7990ac224daffbe50b377bacb0a", "packages": [ { "name": "adhocore/jwt", @@ -3474,16 +3474,16 @@ }, { "name": "utopia-php/audit", - "version": "0.49.0", + "version": "dev-add-batch-logging-method", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d" + "reference": "41d87370f1559656c8695d8376d6ac8406c5bd8e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/41d87370f1559656c8695d8376d6ac8406c5bd8e", + "reference": "41d87370f1559656c8695d8376d6ac8406c5bd8e", "shasum": "" }, "require": { @@ -3515,9 +3515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.49.0" + "source": "https://github.com/utopia-php/audit/tree/add-batch-logging-method" }, - "time": "2025-02-04T07:27:18+00:00" + "time": "2025-02-12T05:16:50+00:00" }, { "name": "utopia-php/cache", @@ -8747,7 +8747,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/audit": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index c0bcab1c3a..795900f9da 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -15,6 +15,11 @@ use Utopia\Queue\Message; class Audits extends Action { + private const BATCH_SIZE = 5_000; + private const BATCH_TIME_WINDOW = 60; + + private static array $pendingEvents = []; + public static function getName(): string { return 'audits'; @@ -44,7 +49,6 @@ class Audits extends Action */ public function action(Message $message, Database $dbForProject): void { - $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -63,23 +67,53 @@ class Audits extends Action $userEmail = $user->getAttribute('email', ''); $userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER); - $audit = new Audit($dbForProject); - $audit->log( - userId: $user->getInternalId(), - // Pass first, most verbose event pattern - event: $event, - resource: $resource, - userAgent: $userAgent, - ip: $ip, - location: '', - data: [ + // Create event data + $eventData = [ + 'userId' => $user->getInternalId(), + 'event' => $event, + 'resource' => $resource, + 'userAgent' => $userAgent, + 'ip' => $ip, + 'location' => '', + 'data' => [ 'userId' => $user->getId(), 'userName' => $userName, 'userEmail' => $userEmail, 'userType' => $userType, 'mode' => $mode, 'data' => $auditPayload, - ] - ); + ], + 'timestamp' => time() + ]; + + self::$pendingEvents[] = $eventData; + + // Check if we should process the batch by checking both for the batch size and the elapsed time + $shouldProcessBatch = count(self::$pendingEvents) >= self::BATCH_SIZE; + if (!$shouldProcessBatch && count(self::$pendingEvents) > 0) { + $oldestEventTime = self::$pendingEvents[0]['timestamp']; + $shouldProcessBatch = (time() - $oldestEventTime) >= self::BATCH_TIME_WINDOW; + } + + if ($shouldProcessBatch) { + $audit = new Audit($dbForProject); + $batchEvents = array_map(function($event) { + return [ + 'userId' => $event['userId'], + 'event' => $event['event'], + 'resource' => $event['resource'], + 'userAgent' => $event['userAgent'], + 'ip' => $event['ip'], + 'location' => $event['location'], + 'data' => $event['data'], + 'timestamp' => $event['timestamp'] + ]; + }, self::$pendingEvents); + + $audit->logByBatch($batchEvents); + + // Clear the pending events after successful batch processing + self::$pendingEvents = []; + } } } From df27e8950bfb43fe379ddfc8a15e649582d4afa1 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 11:02:29 +0530 Subject: [PATCH 2/9] feat: update composer --- .cursorignore | 7 ------- composer.json | 2 +- composer.lock | 18 ++++++++---------- src/Appwrite/Platform/Workers/Audits.php | 2 +- 4 files changed, 10 insertions(+), 19 deletions(-) delete mode 100644 .cursorignore diff --git a/.cursorignore b/.cursorignore deleted file mode 100644 index 95ce4247a6..0000000000 --- a/.cursorignore +++ /dev/null @@ -1,7 +0,0 @@ -docs/* -public/* -vendor/* -dev/* -.vscode/* -app/assets/* -app/sdks/* \ No newline at end of file diff --git a/composer.json b/composer.json index 6e25f5f912..58b74c3fe3 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.49.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "dev-add-batch-logging-method", + "utopia-php/audit": "0.50.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index e77db78d2f..fd89ba3ae8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9b7ba7990ac224daffbe50b377bacb0a", + "content-hash": "884381b7cc6c225f83c397eb472ddf11", "packages": [ { "name": "adhocore/jwt", @@ -3474,16 +3474,16 @@ }, { "name": "utopia-php/audit", - "version": "dev-add-batch-logging-method", + "version": "0.50.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "41d87370f1559656c8695d8376d6ac8406c5bd8e" + "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/41d87370f1559656c8695d8376d6ac8406c5bd8e", - "reference": "41d87370f1559656c8695d8376d6ac8406c5bd8e", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/c0da7dcdd35fc7d3f9640ba21cc82607cf7da729", + "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729", "shasum": "" }, "require": { @@ -3515,9 +3515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/add-batch-logging-method" + "source": "https://github.com/utopia-php/audit/tree/0.50.0" }, - "time": "2025-02-12T05:16:50+00:00" + "time": "2025-02-12T05:30:25+00:00" }, { "name": "utopia-php/cache", @@ -8747,9 +8747,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/audit": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 795900f9da..d7b446ab5d 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -110,7 +110,7 @@ class Audits extends Action ]; }, self::$pendingEvents); - $audit->logByBatch($batchEvents); + $audit->logBatch($batchEvents); // Clear the pending events after successful batch processing self::$pendingEvents = []; From 7f706fce5dd6f962163ec8252487065c2b30dcc6 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 11:09:24 +0530 Subject: [PATCH 3/9] feat: linter & console logs --- src/Appwrite/Platform/Workers/Audits.php | 25 +++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index d7b446ab5d..8848d03645 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -6,6 +6,7 @@ use Appwrite\Auth\Auth; use Exception; use Throwable; use Utopia\Audit\Audit; +use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; @@ -18,7 +19,7 @@ class Audits extends Action private const BATCH_SIZE = 5_000; private const BATCH_TIME_WINDOW = 60; - private static array $pendingEvents = []; + private static array $logs = []; public static function getName(): string { @@ -55,6 +56,8 @@ class Audits extends Action throw new Exception('Missing payload'); } + Console::info('Aggregating audit logs'); + $event = $payload['event'] ?? ''; $auditPayload = $payload['payload'] ?? ''; $mode = $payload['mode'] ?? ''; @@ -86,18 +89,20 @@ class Audits extends Action 'timestamp' => time() ]; - self::$pendingEvents[] = $eventData; + self::$logs[] = $eventData; // Check if we should process the batch by checking both for the batch size and the elapsed time - $shouldProcessBatch = count(self::$pendingEvents) >= self::BATCH_SIZE; - if (!$shouldProcessBatch && count(self::$pendingEvents) > 0) { - $oldestEventTime = self::$pendingEvents[0]['timestamp']; + $shouldProcessBatch = count(self::$logs) >= self::BATCH_SIZE; + if (!$shouldProcessBatch && count(self::$logs) > 0) { + $oldestEventTime = self::$logs[0]['timestamp']; $shouldProcessBatch = (time() - $oldestEventTime) >= self::BATCH_TIME_WINDOW; } if ($shouldProcessBatch) { + Console::log('Processing batch with ' . count(self::$logs) . ' events'); + $audit = new Audit($dbForProject); - $batchEvents = array_map(function($event) { + $batchEvents = array_map(function ($event) { return [ 'userId' => $event['userId'], 'event' => $event['event'], @@ -108,12 +113,14 @@ class Audits extends Action 'data' => $event['data'], 'timestamp' => $event['timestamp'] ]; - }, self::$pendingEvents); + }, self::$logs); $audit->logBatch($batchEvents); - + // Clear the pending events after successful batch processing - self::$pendingEvents = []; + self::$logs = []; + + Console::success('Audit logs processed successfully'); } } } From 978284b36fe6934c62993c9736ba7639fbb49e35 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 11:33:17 +0530 Subject: [PATCH 4/9] feat: use smaller batch size on development environments --- src/Appwrite/Platform/Workers/Audits.php | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 8848d03645..bdd3f942e1 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -8,19 +8,29 @@ use Throwable; use Utopia\Audit\Audit; use Utopia\CLI\Console; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\System\System; class Audits extends Action { - private const BATCH_SIZE = 5_000; - private const BATCH_TIME_WINDOW = 60; + private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + private const BATCH_SIZE_PRODUCTION = 5_000; + private const BATCH_AGGREGATION_INTERVAL = 60; private static array $logs = []; + private function getBatchSize(): int + { + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; + } + public static function getName(): string { return 'audits'; @@ -86,16 +96,17 @@ class Audits extends Action 'mode' => $mode, 'data' => $auditPayload, ], - 'timestamp' => time() + 'timestamp' => DateTime::formatTz(DateTime::now()) ]; self::$logs[] = $eventData; // Check if we should process the batch by checking both for the batch size and the elapsed time - $shouldProcessBatch = count(self::$logs) >= self::BATCH_SIZE; + $batchSize = $this->getBatchSize(); + $shouldProcessBatch = count(self::$logs) >= $batchSize; if (!$shouldProcessBatch && count(self::$logs) > 0) { $oldestEventTime = self::$logs[0]['timestamp']; - $shouldProcessBatch = (time() - $oldestEventTime) >= self::BATCH_TIME_WINDOW; + $shouldProcessBatch = (time() - $oldestEventTime) >= self::BATCH_AGGREGATION_INTERVAL; } if ($shouldProcessBatch) { From 48525ce755996a3814e59fc335cf252e3f4131b0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 11:34:39 +0530 Subject: [PATCH 5/9] chore: linter --- src/Appwrite/Platform/Workers/Audits.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index bdd3f942e1..6c37e3e35f 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -19,15 +19,15 @@ use Utopia\System\System; class Audits extends Action { private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - private const BATCH_SIZE_PRODUCTION = 5_000; + private const BATCH_SIZE_PRODUCTION = 5_000; private const BATCH_AGGREGATION_INTERVAL = 60; private static array $logs = []; private function getBatchSize(): int { - return System::getEnv('_APP_ENV', 'development') === 'development' - ? self::BATCH_SIZE_DEVELOPMENT + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT : self::BATCH_SIZE_PRODUCTION; } From 294f6818faeb627c0504c258eea9899787e44fbb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 12:23:20 +0530 Subject: [PATCH 6/9] chore: review comments --- src/Appwrite/Platform/Workers/Audits.php | 27 ++++++++---------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 6c37e3e35f..47be7a868b 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -113,25 +113,16 @@ class Audits extends Action Console::log('Processing batch with ' . count(self::$logs) . ' events'); $audit = new Audit($dbForProject); - $batchEvents = array_map(function ($event) { - return [ - 'userId' => $event['userId'], - 'event' => $event['event'], - 'resource' => $event['resource'], - 'userAgent' => $event['userAgent'], - 'ip' => $event['ip'], - 'location' => $event['location'], - 'data' => $event['data'], - 'timestamp' => $event['timestamp'] - ]; - }, self::$logs); - $audit->logBatch($batchEvents); - - // Clear the pending events after successful batch processing - self::$logs = []; - - Console::success('Audit logs processed successfully'); + try { + $audit->logBatch(self::$logs); + Console::success('Audit logs processed successfully'); + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + // Clear the pending events after successful batch processing + self::$logs = []; + } } } } From 2dffcb29b09dc628716e22c229c525af992564ef Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 13:09:31 +0530 Subject: [PATCH 7/9] chore: add comment about the unit --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 47be7a868b..01eac10502 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -20,7 +20,7 @@ class Audits extends Action { private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development private const BATCH_SIZE_PRODUCTION = 5_000; - private const BATCH_AGGREGATION_INTERVAL = 60; + private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private static array $logs = []; From 9989137ed463fe32f9fe49aae2261bf386a055f3 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 22:01:28 +0530 Subject: [PATCH 8/9] chore: use private variables for aggregation --- package.json | 8 -------- src/Appwrite/Platform/Workers/Audits.php | 23 +++++++++++++---------- 2 files changed, 13 insertions(+), 18 deletions(-) delete mode 100644 package.json diff --git a/package.json b/package.json deleted file mode 100644 index 6e32c7d515..0000000000 --- a/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "private": true, - "name": "@appwrite.io/repo", - "repository": { - "type": "git", - "url": "git+https://github.com/appwrite/appwrite.git" - } -} \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 01eac10502..f1ae46eea7 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -22,7 +22,9 @@ class Audits extends Action private const BATCH_SIZE_PRODUCTION = 5_000; private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds - private static array $logs = []; + private int $lastTriggeredTime = 0; + + private array $logs = []; private function getBatchSize(): int { @@ -46,6 +48,8 @@ class Audits extends Action ->inject('message') ->inject('dbForProject') ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject)); + + $this->lastTriggeredTime = time(); } @@ -99,29 +103,28 @@ class Audits extends Action 'timestamp' => DateTime::formatTz(DateTime::now()) ]; - self::$logs[] = $eventData; + $this->logs[] = $eventData; // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $shouldProcessBatch = count(self::$logs) >= $batchSize; - if (!$shouldProcessBatch && count(self::$logs) > 0) { - $oldestEventTime = self::$logs[0]['timestamp']; - $shouldProcessBatch = (time() - $oldestEventTime) >= self::BATCH_AGGREGATION_INTERVAL; + $shouldProcessBatch = count($this->logs) >= $batchSize; + if (!$shouldProcessBatch && count($this->logs) > 0) { + $shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } if ($shouldProcessBatch) { - Console::log('Processing batch with ' . count(self::$logs) . ' events'); - + Console::log('Processing batch with ' . count($this->logs) . ' events'); $audit = new Audit($dbForProject); try { - $audit->logBatch(self::$logs); + $audit->logBatch($this->logs); Console::success('Audit logs processed successfully'); } catch (Throwable $e) { Console::error('Error processing audit logs: ' . $e->getMessage()); } finally { // Clear the pending events after successful batch processing - self::$logs = []; + $this->logs = []; + $this->lastTriggeredTime = time(); } } } From a0482aca7a44d5c0adad03e6d47dbb49e551873e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 12 Feb 2025 22:04:46 +0530 Subject: [PATCH 9/9] chore: revert package.json --- package.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 package.json diff --git a/package.json b/package.json new file mode 100644 index 0000000000..6e32c7d515 --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "private": true, + "name": "@appwrite.io/repo", + "repository": { + "type": "git", + "url": "git+https://github.com/appwrite/appwrite.git" + } +} \ No newline at end of file