From c96b6300daa54603518a7198873a457e9f1569c5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 12:39:32 +0300 Subject: [PATCH 01/10] Check audits --- src/Appwrite/Platform/Workers/Audits.php | 38 +++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 04b562a219..869ef7a285 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -18,7 +18,7 @@ use Utopia\System\System; class Audits extends Action { - protected const int BATCH_AGGREGATION_INTERVAL = 60; // in seconds + protected const int BATCH_AGGREGATION_INTERVAL = 10000; // in seconds private int $lastTriggeredTime = 0; @@ -40,6 +40,8 @@ class Audits extends Action */ public function __construct() { + Console::error('__construct Audits'); + $this ->desc('Audits worker') ->inject('message') @@ -71,6 +73,9 @@ class Audits extends Action Console::info('Aggregating audit logs'); + Console::error('project '.$project->getSequence()); + Console::error('lastTriggeredTime '.$this->lastTriggeredTime); + $event = $payload['event'] ?? ''; $auditPayload = ''; @@ -121,27 +126,46 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); + $batchSize = 3; + $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs['logs']), 0); $shouldProcessBatch = $logCount >= $batchSize; + + var_dump('$logCount'); + var_dump($logCount); + var_dump('$shouldProcessBatch 1'); + var_dump($shouldProcessBatch); + if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } + var_dump('$shouldProcessBatch 2'); + var_dump($shouldProcessBatch); + + var_dump($this->logs); + if (!$shouldProcessBatch) { return new NoCommit(); } - try { - foreach ($this->logs as $sequence => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + $logs = $this->logs; + $this->logs = []; + + try { + foreach ($logs as $projectLogs) { + /** + * @var $projectDocument Document + */ + $projectDocument = $projectLogs['project']; + $dbForProject = $getProjectDB($projectDocument); + + Console::log('Processing Project "'.$projectDocument->getSequence().'" batch with ' . count($projectLogs['logs']) . ' events'); - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); $audit = new Audit($dbForProject); $audit->logBatch($projectLogs['logs']); Console::success('Audit logs processed successfully'); - - unset($this->logs[$sequence]); } } catch (Throwable $e) { Console::error('Error processing audit logs: ' . $e->getMessage()); From dffbe2e4abb9d18c639005f7f9fe324b051d679a Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 12:52:51 +0300 Subject: [PATCH 02/10] Check audits --- src/Appwrite/Platform/Workers/Audits.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 869ef7a285..966a17808f 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -158,10 +158,11 @@ class Audits extends Action * @var $projectDocument Document */ $projectDocument = $projectLogs['project']; - $dbForProject = $getProjectDB($projectDocument); Console::log('Processing Project "'.$projectDocument->getSequence().'" batch with ' . count($projectLogs['logs']) . ' events'); + $dbForProject = $getProjectDB($projectDocument); + $audit = new Audit($dbForProject); $audit->logBatch($projectLogs['logs']); @@ -170,7 +171,9 @@ class Audits extends Action } catch (Throwable $e) { Console::error('Error processing audit logs: ' . $e->getMessage()); } + $this->lastTriggeredTime = time(); + return new Commit(); } } From f4caa69840615004246fd20d18b13d2fff246f10 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 13:07:31 +0300 Subject: [PATCH 03/10] Remove dbg --- src/Appwrite/Platform/Workers/Audits.php | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 966a17808f..d2cb55fe20 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -18,7 +18,7 @@ use Utopia\System\System; class Audits extends Action { - protected const int BATCH_AGGREGATION_INTERVAL = 10000; // in seconds + protected const int BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -73,9 +73,6 @@ class Audits extends Action Console::info('Aggregating audit logs'); - Console::error('project '.$project->getSequence()); - Console::error('lastTriggeredTime '.$this->lastTriggeredTime); - $event = $payload['event'] ?? ''; $auditPayload = ''; @@ -126,25 +123,14 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $batchSize = 3; $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs['logs']), 0); $shouldProcessBatch = $logCount >= $batchSize; - var_dump('$logCount'); - var_dump($logCount); - var_dump('$shouldProcessBatch 1'); - var_dump($shouldProcessBatch); - if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - var_dump('$shouldProcessBatch 2'); - var_dump($shouldProcessBatch); - - var_dump($this->logs); - if (!$shouldProcessBatch) { return new NoCommit(); } From 52141b825c7326d80887affa4861be75b7861eb1 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 13:08:24 +0300 Subject: [PATCH 04/10] Remove dbg --- src/Appwrite/Platform/Workers/Audits.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index d2cb55fe20..e52ad613ed 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -40,8 +40,6 @@ class Audits extends Action */ public function __construct() { - Console::error('__construct Audits'); - $this ->desc('Audits worker') ->inject('message') From 97ce2bdc3a31823cfb315592fde3b3da0c0f5fff Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 13:09:23 +0300 Subject: [PATCH 05/10] Lines --- src/Appwrite/Platform/Workers/Audits.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index e52ad613ed..6fa81c5772 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -121,10 +121,8 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs['logs']), 0); $shouldProcessBatch = $logCount >= $batchSize; - if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } From ad4e3be80c2a0b7003e5bb155776c9846f3b93e6 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 13:10:04 +0300 Subject: [PATCH 06/10] Lines --- src/Appwrite/Platform/Workers/Audits.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 6fa81c5772..c99c5eafe4 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -153,9 +153,7 @@ class Audits extends Action } catch (Throwable $e) { Console::error('Error processing audit logs: ' . $e->getMessage()); } - $this->lastTriggeredTime = time(); - return new Commit(); } } From bd8d6fdc59e43af2d3e6461e401b2713cf042c26 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 1 Sep 2025 18:35:12 +0300 Subject: [PATCH 07/10] Try catch inside --- src/Appwrite/Platform/Workers/Audits.php | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index c99c5eafe4..e8ae2f2168 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -134,25 +134,19 @@ class Audits extends Action $logs = $this->logs; $this->logs = []; - try { - foreach ($logs as $projectLogs) { - /** - * @var $projectDocument Document - */ + foreach ($logs as $projectLogs) { + try { $projectDocument = $projectLogs['project']; - - Console::log('Processing Project "'.$projectDocument->getSequence().'" batch with ' . count($projectLogs['logs']) . ' events'); - + Console::log('Processing Project "' . $projectDocument->getSequence() . '" batch with ' . count($projectLogs['logs']) . ' events'); $dbForProject = $getProjectDB($projectDocument); - $audit = new Audit($dbForProject); - $audit->logBatch($projectLogs['logs']); Console::success('Audit logs processed successfully'); + } catch (Throwable $e) { + Console::error('Error processing audit logs for Project "' . $projectDocument->getSequence() . '": ' . $e->getMessage()); } - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); } + $this->lastTriggeredTime = time(); return new Commit(); } From 0af3734b6f20c6f19be97dd30ee283b6836798c8 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 13:41:10 +0300 Subject: [PATCH 08/10] finally unset --- src/Appwrite/Platform/Workers/Audits.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index e8ae2f2168..4f6485efe5 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -131,19 +131,19 @@ class Audits extends Action return new NoCommit(); } - $logs = $this->logs; - $this->logs = []; - - foreach ($logs as $projectLogs) { + foreach ($this->logs as $sequence => $projectLogs) { try { + Console::log('Processing Project "' . $sequence . '" batch with ' . count($projectLogs['logs']) . ' events'); + $projectDocument = $projectLogs['project']; - Console::log('Processing Project "' . $projectDocument->getSequence() . '" batch with ' . count($projectLogs['logs']) . ' events'); $dbForProject = $getProjectDB($projectDocument); $audit = new Audit($dbForProject); $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); + } catch (Throwable $e) { Console::error('Error processing audit logs for Project "' . $projectDocument->getSequence() . '": ' . $e->getMessage()); + } finally { + unset($this->logs[$sequence]); } } From c2adf79511ea8af9953279b46ec42ab82a3b7e2c Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 13:43:42 +0300 Subject: [PATCH 09/10] message --- src/Appwrite/Platform/Workers/Audits.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 4f6485efe5..7316f5c8a9 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -140,6 +140,7 @@ class Audits extends Action $audit = new Audit($dbForProject); $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); } catch (Throwable $e) { Console::error('Error processing audit logs for Project "' . $projectDocument->getSequence() . '": ' . $e->getMessage()); } finally { From 711d31b56536650c252e37d9a303c19c534fa408 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 13:48:17 +0300 Subject: [PATCH 10/10] Console::error message fix --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 7316f5c8a9..a88e2e641f 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -142,7 +142,7 @@ class Audits extends Action Console::success('Audit logs processed successfully'); } catch (Throwable $e) { - Console::error('Error processing audit logs for Project "' . $projectDocument->getSequence() . '": ' . $e->getMessage()); + Console::error('Error processing audit logs for Project "' . $sequence . '": ' . $e->getMessage()); } finally { unset($this->logs[$sequence]); }