diff --git a/.env b/.env index 88dec63b1c..c301c53123 100644 --- a/.env +++ b/.env @@ -102,6 +102,7 @@ _APP_STATS_RESOURCES_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_INTERVAL_DOMAIN_VERIFICATION=60 +_APP_INTERVAL_CLEANUP_STALE_EXECUTIONS=300 _APP_USAGE_STATS=enabled _APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG_REALTIME= diff --git a/docker-compose.yml b/docker-compose.yml index 20c0ad8f79..c5b88a2174 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -880,6 +880,7 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - _APP_INTERVAL_DOMAIN_VERIFICATION + - _APP_INTERVAL_CLEANUP_STALE_EXECUTIONS appwrite-task-stats-resources: container_name: appwrite-task-stats-resources diff --git a/src/Appwrite/Platform/Tasks/Interval.php b/src/Appwrite/Platform/Tasks/Interval.php index 9d3d782501..0985447c2e 100644 --- a/src/Appwrite/Platform/Tasks/Interval.php +++ b/src/Appwrite/Platform/Tasks/Interval.php @@ -24,22 +24,30 @@ class Interval extends Action $this ->desc('Schedules tasks on regular intervals by publishing them to our queues') ->inject('dbForPlatform') + ->inject('getProjectDB') ->inject('queueForCertificates') ->callback($this->action(...)); } - public function action(Database $dbForPlatform, Certificate $queueForCertificates): void + public function action(Database $dbForPlatform, callable $getProjectDB, Certificate $queueForCertificates): void { Console::title('Interval V1'); Console::success(APP_NAME . ' interval process v1 has started'); $intervalDomainVerification = (int) System::getEnv('_APP_INTERVAL_DOMAIN_VERIFICATION', '60'); // 1 minute + $intervalCleanupStaleExecutions = (int) System::getEnv('_APP_INTERVAL_CLEANUP_STALE_EXECUTIONS', '300'); // 5 minutes \go(function () use ($dbForPlatform, $queueForCertificates, $intervalDomainVerification) { Console::loop(function () use ($dbForPlatform, $queueForCertificates) { $this->verifyDomain($dbForPlatform, $queueForCertificates); }, $intervalDomainVerification); }); + + \go(function () use ($dbForPlatform, $getProjectDB, $intervalCleanupStaleExecutions) { + Console::loop(function () use ($dbForPlatform, $getProjectDB) { + $this->cleanupStaleExecutions($dbForPlatform, $getProjectDB); + }, $intervalCleanupStaleExecutions); + }); } private function verifyDomain(Database $dbForPlatform, Certificate $queueForCertificates): void @@ -72,4 +80,47 @@ class Interval extends Action ->trigger(); } } + + private function cleanupStaleExecutions(Database $dbForPlatform, callable $getProjectDB): void + { + $time = DatabaseDateTime::now(); + $staleThreshold = DatabaseDateTime::addSeconds(new DateTime(), -1200); // 20 minutes ago + + Console::info("[{$time}] Starting cleanup of stale executions"); + + $dbForPlatform->foreach( + 'projects', + function (Document $project) use ($getProjectDB, $time, $staleThreshold) { + try { + $dbForProject = $getProjectDB($project); + + $staleExecutions = $dbForProject->find('executions', [ + Query::equal('status', ['processing']), + Query::lessThan('$createdAt', $staleThreshold), + Query::limit(100), + ]); + + if (\count($staleExecutions) === 0) { + return; + } + + Console::info("[{$time}] Found " . \count($staleExecutions) . " stale executions in project {$project->getId()}"); + + foreach ($staleExecutions as $execution) { + $execution->setAttribute('status', 'failed'); + $execution->setAttribute('errors', 'Execution timed out'); + $dbForProject->updateDocument('executions', $execution->getId(), $execution); + } + } catch (\Throwable $th) { + Console::error("[{$time}] Failed to cleanup stale executions for project {$project->getId()}: " . $th->getMessage()); + } + }, + [ + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::limit(100), + ] + ); + + Console::info("[{$time}] Completed cleanup of stale executions"); + } }