From 5eb7c36f50e2713aee76beaff1bb207eca130116 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 10:32:36 +0000 Subject: [PATCH 1/7] chore: add console audit retention --- .env | 2 +- app/worker.php | 2 +- docker-compose.yml | 1 + src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 1893e023ba..c68f0a0d52 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/worker.php b/app/worker.php index eeefe80000..5d7bd4de41 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,7 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); + return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index b6dc80df6a..af6a323ec8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,6 +389,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES + - _APP_MAINTENANCE_RETENTION_AUDIT appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..75380ed3fe 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -751,9 +751,16 @@ class Deletes extends Action $projectId = $project->getId(); $dbForProject = $getProjectDB($project); + [$projectAuditRetention, $consoleAuditRetention] = array_map( + function ($part) { + return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); + }, + explode(',', $auditRetention) + ); + try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', $auditRetention), + Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 9d8289221925ddf7eeebbd42f56ba9be311dd2a0 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 11:32:06 +0000 Subject: [PATCH 2/7] chore: update logic --- app/worker.php | 10 +++++++++- src/Appwrite/Platform/Workers/Deletes.php | 15 ++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/app/worker.php b/app/worker.php index 5d7bd4de41..18b51eda25 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,15 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months + return array_map( + function ($part) { + [$key, $value] = explode('=', $part); + return [ + $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) + ]; + }, + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + ); }); Server::setResource('executionRetention', function () { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 75380ed3fe..93f8748e49 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log) => $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -746,21 +746,14 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void + private function deleteAuditLogs(Document $project, callable $getProjectDB, array $auditRetention): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - [$projectAuditRetention, $consoleAuditRetention] = array_map( - function ($part) { - return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); - }, - explode(',', $auditRetention) - ); - try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), + Query::lessThan('time', ($projectId === 'console' ? $auditRetention['console'] : $auditRetention['project'])), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 7c3f8504bb65e598f50d07912b5d753c86f5e1f6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:15:14 +0000 Subject: [PATCH 3/7] chore: fix initialization --- app/worker.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/worker.php b/app/worker.php index 18b51eda25..e305ec42d6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,14 +219,14 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_map( - function ($part) { + return array_reduce( + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + function ($carry, $part) { [$key, $value] = explode('=', $part); - return [ - $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) - ]; + $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); + return $carry; }, - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + [] ); }); From b0b57935da0e54074766a3f9a144f41fe2081a55 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:42:19 +0000 Subject: [PATCH 4/7] chore: add comments --- app/worker.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/worker.php b/app/worker.php index e305ec42d6..d0fa5cf4f8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -215,12 +215,12 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { }, ['pools', 'cache']); Server::setResource('abuseRetention', function () { - return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); + return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day }); Server::setResource('auditRetention', function () { return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months function ($carry, $part) { [$key, $value] = explode('=', $part); $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); @@ -231,7 +231,7 @@ Server::setResource('auditRetention', function () { }); Server::setResource('executionRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); // 14 days }); Server::setResource('cache', function (Registry $register) { From 13750f7dbd5b5850075eefab4bacf3e782752004 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:42:10 +0000 Subject: [PATCH 5/7] feat: introduce new env --- .env | 3 ++- app/config/variables.php | 11 ++++++++++- app/views/install/compose.phtml | 3 +++ app/worker.php | 13 ++++--------- docker-compose.yml | 3 +++ 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.env b/.env index c68f0a0d52..c10c12613b 100644 --- a/.env +++ b/.env @@ -85,7 +85,8 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 +_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/config/variables.php b/app/config/variables.php index 98dd9ffec1..27463d2fee 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1048,13 +1048,22 @@ return [ ], [ 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT', - 'description' => 'IThe maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', + 'description' => 'The maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', 'introduction' => '0.7.0', 'default' => '1209600', 'required' => false, 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', + 'description' => 'The maximum duration (in seconds) upto which to retain console audit logs. The default value is 15778800 seconds (6 months).', + 'introduction' => '1.6.2', + 'default' => '15778800', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_MAINTENANCE_RETENTION_ABUSE', 'description' => 'The maximum duration (in seconds) upto which to retain abuse logs. The default value is 86400 seconds (1 day).', diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index fd05d2a0b6..7dfe14fcef 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -148,6 +148,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -340,6 +341,7 @@ $image = $this->getParam('image', ''); - _APP_EXECUTOR_HOST - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES @@ -651,6 +653,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES diff --git a/app/worker.php b/app/worker.php index d0fa5cf4f8..3f99089eef 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,15 +219,10 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months - function ($carry, $part) { - [$key, $value] = explode('=', $part); - $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); - return $carry; - }, - [] - ); + return [ + 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + ]; }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index af6a323ec8..05ddba967a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,6 +171,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -390,6 +391,7 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE appwrite-worker-databases: entrypoint: worker-databases @@ -722,6 +724,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY From 7c795da72fd7aa05b280f27012f87357a08d2814 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:48:50 +0000 Subject: [PATCH 6/7] chore: fix type --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 3f99089eef..744c758c69 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) ]; }); From c0c0c921dc249f2f235a9bd7e8efdce66ae3c137 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 08:08:30 +0000 Subject: [PATCH 7/7] chore: fix naming --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 744c758c69..491f266a18 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), // 14 days + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) // 6 months ]; });