From d009b8ab89274e368fe9c87cc4284342921447ee Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 27 Aug 2021 17:22:52 +0545 Subject: [PATCH 1/7] delete older usage stats --- app/init.php | 1 + app/tasks/maintenance.php | 11 +++++++++++ app/workers/deletes.php | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/app/init.php b/app/init.php index c8261ffd33..475bb3e40d 100644 --- a/app/init.php +++ b/app/init.php @@ -89,6 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; +const DELETE_TYPE_USAGE_STATS = 'certificates'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index eccdf61b17..a5cf63f821 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -39,11 +39,22 @@ $cli ]); } + function notifyDeleteUsageStats(int $interval30m, int $interval1d) + { + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_USAGE_STATS, + 'timestamp1d' => $interval1d, + 'timestamp30m' => $interval30m, + ]); + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); + $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours + $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ $time = date('d-m-Y H:i:s', time()); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f0e66d2b46..fe0037056c 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -72,6 +72,9 @@ class DeletesV1 extends Worker $document = new Document($this->args['document']); $this->deleteCertificates($document); break; + + case DELETE_TYPE_USAGE_STATS: + $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); default: Console::error('No delete operation for type: '.$type); @@ -82,6 +85,29 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param int $timestamp1d + * @param int $timestamp30m + */ + protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) { + $this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) { + if (!($dbForInternal = $this->getInternalDB($projectId))) { + throw new Exception('Failed to get projectDB for project '.$projectId); + } + + // Delete Usage stats + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp1d]), + new Query('period', Query::TYPE_EQUAL, ['1d', '15m']), + ], $dbForInternal); + + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp30m]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + ], $dbForInternal); + }); + } /** * @param Document $document teams document From 3a29d6a565487d535a96860312a420ceb83c85c4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Sep 2021 12:16:03 +0545 Subject: [PATCH 2/7] fix deletes, remove 15m period after refactor --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 4d5bd0cacc..3142065fa5 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -105,7 +105,7 @@ class DeletesV1 extends Worker // Delete Usage stats $this->deleteByGroup('stats', [ new Query('time', Query::TYPE_LESSER, [$timestamp1d]), - new Query('period', Query::TYPE_EQUAL, ['1d', '15m']), + new Query('period', Query::TYPE_EQUAL, ['1d']), ], $dbForInternal); $this->deleteByGroup('stats', [ From 334a59e3c1e226a648c4f15f7e5aa28d6e2773ba Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Sep 2021 12:17:32 +0545 Subject: [PATCH 3/7] schedule usage stats deletion --- app/tasks/maintenance.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a5cf63f821..f30c2a9a42 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -56,11 +56,12 @@ $cli $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ + Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); }, $interval); }); \ No newline at end of file From 1a21c8fdbbf53fd0ff94622244895a470016325e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 6 Sep 2021 11:48:44 +0545 Subject: [PATCH 4/7] fix constant value --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 475bb3e40d..78c44076e5 100644 --- a/app/init.php +++ b/app/init.php @@ -89,7 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -const DELETE_TYPE_USAGE_STATS = 'certificates'; +const DELETE_TYPE_USAGE_STATS = 'usageStats'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; From 9f93ceec13cdb962d4c604bdc6ced7d046eefc34 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 6 Sep 2021 15:24:36 +0545 Subject: [PATCH 5/7] missing break --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3142065fa5..f9571949f2 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -81,7 +81,7 @@ class DeletesV1 extends Worker case DELETE_TYPE_USAGE_STATS: $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); - + break; default: Console::error('No delete operation for type: '.$type); break; From 1ecd9eecffab2f804140e89c21a8bdc548063950 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 7 Sep 2021 11:03:30 +0545 Subject: [PATCH 6/7] fix deletes interval --- app/tasks/maintenance.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index f30c2a9a42..fdfd7c942b 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -43,8 +43,8 @@ $cli { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_USAGE_STATS, - 'timestamp1d' => $interval1d, - 'timestamp30m' => $interval30m, + 'timestamp1d' => time() - $interval1d, + 'timestamp30m' => time() - $interval30m, ]); } From b38c77255e0d03e4e5206a5da09ede133d7ec508 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 8 Sep 2021 12:40:48 +0545 Subject: [PATCH 7/7] Update app/init.php Co-authored-by: Eldad A. Fux --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 78c44076e5..4c1e6cf219 100644 --- a/app/init.php +++ b/app/init.php @@ -89,7 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -const DELETE_TYPE_USAGE_STATS = 'usageStats'; +const DELETE_TYPE_USAGE = 'usage'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery';