Merge pull request #11147 from appwrite/executions-cleanup

2026-05-23 00:49:02 +00:00 · 2026-01-16 16:33:39 +05:30 · 2026-01-16 16:33:39 +05:30 · 429f73eb85
commit 429f73eb85
parent ce91b1a03d ccaea5d010
7 changed files with 175 additions and 21 deletions
--- a/app/config/collections/projects.php
+++ b/app/config/collections/projects.php
@ -2098,6 +2098,13 @@ return [
                'lengths' => [],
                'orders' => [],
            ],
+            [
+                '$id' => ID::custom('_key_resourceType'),
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['resourceType'],
+                'lengths' => [],
+                'orders' => [Database::ORDER_ASC],
+            ],
        ],
    ],

--- a/app/controllers/general.php
+++ b/app/controllers/general.php
@ -6,6 +6,7 @@ use Ahc\Jwt\JWT;
 use Ahc\Jwt\JWTException;
 use Appwrite\Auth\Key;
 use Appwrite\Event\Certificate;
+use Appwrite\Event\Delete as DeleteEvent;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\StatsUsage;
@ -59,7 +60,7 @@ Config::setParam('domainVerification', false);
 Config::setParam('cookieDomain', 'localhost');
 Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE);

-function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey)
+function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeleteEvent $queueForDeletes, int $executionsRetentionCount)
 {
    $host = $request->getHostname() ?? '';
    if (!empty($previewHostname)) {
@ -802,6 +803,20 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw
            ->setProject($project)
            ->trigger();

+        /* cleanup */
+        if ($executionsRetentionCount > 0 && ENABLE_EXECUTIONS_LIMIT_ON_ROUTE) {
+            $resourceType = $type === 'function'
+                ? RESOURCE_TYPE_FUNCTIONS
+                : RESOURCE_TYPE_SITES;
+
+            $queueForDeletes
+                ->setProject($project)
+                ->setResourceType($resourceType)
+                ->setResource($resource->getSequence())
+                ->setType(DELETE_TYPE_EXECUTIONS_LIMIT)
+                ->trigger();
+        }
+
        return true;
    } elseif ($type === 'api') {
        return false;
@ -812,8 +827,6 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw
    } else {
        throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Unknown resource type ' . $type, view: $errorView);
    }
-
-    return false;
 }

 App::init()
@ -863,7 +876,9 @@ App::init()
    ->inject('apiKey')
    ->inject('cors')
    ->inject('authorization')
-    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Func $queueForFunctions, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization) {
+    ->inject('queueForDeletes')
+    ->inject('executionsRetentionCount')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Func $queueForFunctions, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) {
        /*
        * Appwrite Router
        */
@ -871,7 +886,7 @@ App::init()
        $platformHostnames = $platform['hostnames'] ?? [];
        // Only run Router when external domain
        if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) {
-            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) {
+            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) {
                $utopia->getRoute()?->label('router', true);
            }
        }
@ -1144,14 +1159,16 @@ App::options()
    ->inject('apiKey')
    ->inject('cors')
    ->inject('authorization')
-    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization) {
+    ->inject('queueForDeletes')
+    ->inject('executionsRetentionCount')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) {
        /*
        * Appwrite Router
        */
        $platformHostnames = $platform['hostnames'] ?? [];
        // Only run Router when external domain
        if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) {
-            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) {
+            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) {
                $utopia->getRoute()?->label('router', true);
            }
        }
@ -1535,13 +1552,15 @@ App::get('/robots.txt')
    ->inject('previewHostname')
    ->inject('apiKey')
    ->inject('authorization')
-    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization) {
+    ->inject('queueForDeletes')
+    ->inject('executionsRetentionCount')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) {
        $platformHostnames = $platform['hostnames'] ?? [];
        if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) {
            $template = new View(__DIR__ . '/../views/general/robots.phtml');
            $response->text($template->render(false));
        } else {
-            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) {
+            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) {
                $utopia->getRoute()?->label('router', true);
            }
        }
@ -1568,13 +1587,15 @@ App::get('/humans.txt')
    ->inject('previewHostname')
    ->inject('apiKey')
    ->inject('authorization')
-    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization) {
+    ->inject('queueForDeletes')
+    ->inject('executionsRetentionCount')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) {
        $platformHostnames = $platform['hostnames'] ?? [];
        if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) {
            $template = new View(__DIR__ . '/../views/general/humans.phtml');
            $response->text($template->render(false));
        } else {
-            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) {
+            if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) {
                $utopia->getRoute()?->label('router', true);
            }
        }
--- a/app/init/constants.php
+++ b/app/init/constants.php
@ -181,8 +181,10 @@ const BUILD_TYPE_DEPLOYMENT = 'deployment';
 const BUILD_TYPE_RETRY = 'retry';

 // Deletion Types
-const DELETE_TYPE_DATABASES = 'databases';

+const ENABLE_EXECUTIONS_LIMIT_ON_ROUTE = false;
+
+const DELETE_TYPE_DATABASES = 'databases';
 const DELETE_TYPE_DOCUMENT = 'document';
 const DELETE_TYPE_COLLECTIONS = 'collections';
 const DELETE_TYPE_TRANSACTION = 'transaction';
@ -194,6 +196,7 @@ const DELETE_TYPE_DEPLOYMENTS = 'deployments';
 const DELETE_TYPE_USERS = 'users';
 const DELETE_TYPE_TEAM_PROJECTS = 'teams_projects';
 const DELETE_TYPE_EXECUTIONS = 'executions';
+const DELETE_TYPE_EXECUTIONS_LIMIT = 'executionsLimit';
 const DELETE_TYPE_AUDIT = 'audit';
 const DELETE_TYPE_ABUSE = 'abuse';
 const DELETE_TYPE_USAGE = 'usage';
--- a/app/init/resources.php
+++ b/app/init/resources.php
@ -1156,3 +1156,11 @@ App::setResource('resourceToken', function ($project, $dbForProject, $request, A
 App::setResource('transactionState', function (Database $dbForProject, Authorization $authorization) {
    return new TransactionState($dbForProject, $authorization);
 }, ['dbForProject', 'authorization']);
+
+App::setResource('executionsRetentionCount', function (Document $project, array $plan) {
+    if ($project->getId() === 'console' || empty($plan)) {
+        return 0;
+    }
+
+    return (int) ($plan['executionsRetentionCount'] ?? 100);
+}, ['project', 'plan']);
--- a/app/worker.php
+++ b/app/worker.php
@ -490,6 +490,14 @@ Server::setResource('getAudit', function (Database $dbForPlatform, callable $get
    };
 }, ['dbForPlatform', 'getProjectDB']);

+Server::setResource('executionsRetentionCount', function (Document $project, array $plan) {
+    if ($project->getId() === 'console' || empty($plan)) {
+        return 0;
+    }
+
+    return (int) ($plan['executionsRetentionCount'] ?? 100);
+}, ['project', 'plan']);
+
 $pools = $register->get('pools');
 $platform = new Appwrite();
 $args = $platform->getEnv('argv');
--- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php
+++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php
@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Modules\Functions\Http\Executions;

 use Ahc\Jwt\JWT;
+use Appwrite\Event\Delete as DeleteEvent;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\StatsUsage;
@ -62,7 +63,6 @@ class Create extends Base
            ->label('scope', 'execution.write')
            ->label('resourceType', RESOURCE_TYPE_FUNCTIONS)
            ->label('event', 'functions.[functionId].executions.[executionId].create')
-            ->label('resourceType', RESOURCE_TYPE_FUNCTIONS)
            ->label('sdk', new Method(
                namespace: 'functions',
                group: 'executions',
@ -101,6 +101,8 @@ class Create extends Base
            ->inject('executor')
            ->inject('platform')
            ->inject('authorization')
+            ->inject('queueForDeletes')
+            ->inject('executionsRetentionCount')
            ->callback($this->action(...));
    }

@ -127,6 +129,8 @@ class Create extends Base
        Executor $executor,
        array $platform,
        Authorization $authorization,
+        DeleteEvent $queueForDeletes,
+        int $executionsRetentionCount,
    ) {
        $async = \strval($async) === 'true' || \strval($async) === '1';

@ -513,6 +517,16 @@ class Create extends Base
            }
        }

+        /* cleanup */
+        if ($executionsRetentionCount > 0) {
+            $queueForDeletes
+                ->setProject($project)
+                ->setResource($function->getSequence())
+                ->setResourceType(RESOURCE_TYPE_FUNCTIONS)
+                ->setType(DELETE_TYPE_EXECUTIONS_LIMIT)
+                ->trigger();
+        }
+
        $response
            ->setStatusCode(Response::STATUS_CODE_CREATED)
            ->dynamic($execution, Response::MODEL_EXECUTION);
--- a/src/Appwrite/Platform/Workers/Deletes.php
+++ b/src/Appwrite/Platform/Workers/Deletes.php
@ -30,6 +30,8 @@ use Utopia\Queue\Message;
 use Utopia\Storage\Device;
 use Utopia\System\System;

+use function Swoole\Coroutine\batch;
+
 class Deletes extends Action
 {
    protected array $selects = ['$sequence', '$id', '$collection', '$permissions', '$updatedAt'];
@ -59,6 +61,7 @@ class Deletes extends Action
            ->inject('certificates')
            ->inject('executor')
            ->inject('executionRetention')
+            ->inject('executionsRetentionCount')
            ->inject('auditRetention')
            ->inject('log')
            ->inject('getAudit')
@ -83,6 +86,7 @@ class Deletes extends Action
        CertificatesAdapter $certificates,
        Executor $executor,
        string $executionRetention,
+        int $executionsRetentionCount,
        string $auditRetention,
        Log $log,
        callable $getAudit,
@ -144,6 +148,19 @@ class Deletes extends Action
            case DELETE_TYPE_EXECUTIONS:
                $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
                break;
+            case DELETE_TYPE_EXECUTIONS_LIMIT:
+                $resourceInternalId = $payload['resource'] ?? null;
+                $resourceType = $payload['resourceType'] ?? null;
+                if ($resourceInternalId) {
+                    $this->deleteExecutionsByLimit(
+                        $project,
+                        $getProjectDB,
+                        $executionsRetentionCount,
+                        $resourceInternalId,
+                        $resourceType
+                    );
+                }
+                break;
            case DELETE_TYPE_AUDIT:
                if (!$project->isEmpty()) {
                    $this->deleteAuditLogs($project, $getAudit, $auditRetention);
@ -184,7 +201,7 @@ class Deletes extends Action
                break;
            case DELETE_TYPE_MAINTENANCE:
                $this->deleteExpiredTargets($project, $getProjectDB);
-                $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
+                $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention, $executionsRetentionCount);
                $this->deleteAuditLogs($project, $getAudit, $auditRetention);
                $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime);
                $this->deleteExpiredSessions($project, $getProjectDB);
@ -199,16 +216,15 @@ class Deletes extends Action
     * @param Database $dbForPlatform
     * @param callable $getProjectDB
     * @param string $datetime
-     * @param Document|null $document
     * @return void
     * @throws Conflict
     * @throws Restricted
     * @throws Structure
-     * @throws DatabaseException
+     * @throws DatabaseException|Exception
     */
    private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void
    {
-        // Temporarly accepting both 'fra' and 'default'
+        // Temporarily accepting both 'fra' and 'default'
        // When all migrated, only use _APP_REGION with 'default' as default value
        $regions = [System::getEnv('_APP_REGION', 'default')];
        if (!in_array('default', $regions)) {
@ -694,14 +710,16 @@ class Deletes extends Action
    }

    /**
-     * @param database $dbForPlatform
+     * @param Document $project
     * @param callable $getProjectDB
     * @param string $datetime
+     * @param int|null $executionsRetentionCount
     * @return void
-     * @throws Exception
+     * @throws Exception|DatabaseException
     */
-    private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
+    private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime, ?int $executionsRetentionCount = 0): void
    {
+        /** @var Database $dbForProject */
        $dbForProject = $getProjectDB($project);

        // Delete Executions
@ -711,10 +729,85 @@ class Deletes extends Action
            Query::orderDesc('$createdAt'),
            Query::orderDesc(),
        ], $dbForProject);
+
+        /* delete based on custom retention, if any */
+        $this->deleteExecutionsByLimit($project, $getProjectDB, $executionsRetentionCount);
    }

    /**
-     * @param Database $dbForPlatform
+     * @param Document $project
+     * @param callable $getProjectDB
+     * @param int|null $executionsRetentionCount
+     * @param string|null $resourceInternalId
+     * @param string|null $resourceType
+     * @return void
+     * @throws DatabaseException
+     */
+    protected function deleteExecutionsByLimit(
+        Document $project,
+        callable $getProjectDB,
+        ?int $executionsRetentionCount = 0,
+        ?string $resourceInternalId = null,
+        ?string $resourceType = null
+    ): void {
+        if ($executionsRetentionCount <= 0) {
+            return;
+        }
+
+        /** @var Database $dbForProject */
+        $dbForProject = $getProjectDB($project);
+
+        /* delete log for a given $resourceInternalId  */
+        $delete = function (Database $dbForProject, string $resourceInternalId, string $resourceType) use ($executionsRetentionCount) {
+            // get the execution at position `N+1`
+            $execution = $dbForProject->findOne('executions', [
+                Query::select(['$createdAt']),
+                Query::equal('resourceInternalId', [$resourceInternalId]),
+                Query::equal('resourceType', [$resourceType]),
+                Query::orderDesc('$createdAt'),
+                Query::offset($executionsRetentionCount),
+            ]);
+
+            if (!$execution->isEmpty()) {
+                // delete everything older
+                $cutoffTime = $execution->getAttribute('$createdAt');
+
+                $this->deleteByGroup('executions', [
+                    Query::select([...$this->selects, '$createdAt']),
+                    Query::equal('resourceInternalId', [$resourceInternalId]),
+                    Query::equal('resourceType', [$resourceType]),
+                    Query::lessThan('$createdAt', $cutoffTime),
+                    Query::orderDesc('$createdAt'),
+                    Query::orderDesc(),
+                ], $dbForProject);
+            }
+        };
+
+        if (!empty($resourceInternalId)) {
+            // fast path, no need to list anything!
+            $delete($dbForProject, $resourceInternalId, $resourceType);
+        } else {
+            $processResource = function (string $type) use ($dbForProject, $delete, $resourceType) {
+                $this->listByGroup(
+                    collection: $type,
+                    queries: [Query::select(['$id', '$sequence'])],
+                    database: $dbForProject,
+                    callback: function (Document $resource) use ($dbForProject, $delete, $type) {
+                        $delete($dbForProject, $resource->getSequence(), $type);
+                    }
+                );
+            };
+
+            /* perform processing in parallel */
+            batch([
+                fn () => $processResource(RESOURCE_TYPE_SITES),
+                fn () => $processResource(RESOURCE_TYPE_FUNCTIONS),
+            ]);
+        }
+    }
+
+    /**
+     * @param Document $project
     * @param callable $getProjectDB
     * @return void
     * @throws Exception|Throwable