diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index dae0337dc9..86346d2672 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -2098,6 +2098,13 @@ return [ 'lengths' => [], 'orders' => [], ], + [ + '$id' => ID::custom('_key_resourceType'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceType'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], ], ], diff --git a/app/controllers/general.php b/app/controllers/general.php index e335f284b7..8fc5a11503 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -6,6 +6,7 @@ use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; use Appwrite\Event\Certificate; +use Appwrite\Event\Delete as DeleteEvent; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\StatsUsage; @@ -59,7 +60,7 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey) +function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { $host = $request->getHostname() ?? ''; if (!empty($previewHostname)) { @@ -802,6 +803,20 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw ->setProject($project) ->trigger(); + /* cleanup */ + if ($executionsRetentionCount > 0 && ENABLE_EXECUTIONS_LIMIT_ON_ROUTE) { + $resourceType = $type === 'function' + ? RESOURCE_TYPE_FUNCTIONS + : RESOURCE_TYPE_SITES; + + $queueForDeletes + ->setProject($project) + ->setResourceType($resourceType) + ->setResource($resource->getSequence()) + ->setType(DELETE_TYPE_EXECUTIONS_LIMIT) + ->trigger(); + } + return true; } elseif ($type === 'api') { return false; @@ -812,8 +827,6 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw } else { throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Unknown resource type ' . $type, view: $errorView); } - - return false; } App::init() @@ -863,7 +876,9 @@ App::init() ->inject('apiKey') ->inject('cors') ->inject('authorization') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Func $queueForFunctions, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization) { + ->inject('queueForDeletes') + ->inject('executionsRetentionCount') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Func $queueForFunctions, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { /* * Appwrite Router */ @@ -871,7 +886,7 @@ App::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } @@ -1144,14 +1159,16 @@ App::options() ->inject('apiKey') ->inject('cors') ->inject('authorization') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization) { + ->inject('queueForDeletes') + ->inject('executionsRetentionCount') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } @@ -1535,13 +1552,15 @@ App::get('/robots.txt') ->inject('previewHostname') ->inject('apiKey') ->inject('authorization') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization) { + ->inject('queueForDeletes') + ->inject('executionsRetentionCount') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } @@ -1568,13 +1587,15 @@ App::get('/humans.txt') ->inject('previewHostname') ->inject('apiKey') ->inject('authorization') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization) { + ->inject('queueForDeletes') + ->inject('executionsRetentionCount') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } diff --git a/app/init/constants.php b/app/init/constants.php index d51cb6b7af..e05f31e078 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -181,8 +181,10 @@ const BUILD_TYPE_DEPLOYMENT = 'deployment'; const BUILD_TYPE_RETRY = 'retry'; // Deletion Types -const DELETE_TYPE_DATABASES = 'databases'; +const ENABLE_EXECUTIONS_LIMIT_ON_ROUTE = false; + +const DELETE_TYPE_DATABASES = 'databases'; const DELETE_TYPE_DOCUMENT = 'document'; const DELETE_TYPE_COLLECTIONS = 'collections'; const DELETE_TYPE_TRANSACTION = 'transaction'; @@ -194,6 +196,7 @@ const DELETE_TYPE_DEPLOYMENTS = 'deployments'; const DELETE_TYPE_USERS = 'users'; const DELETE_TYPE_TEAM_PROJECTS = 'teams_projects'; const DELETE_TYPE_EXECUTIONS = 'executions'; +const DELETE_TYPE_EXECUTIONS_LIMIT = 'executionsLimit'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_USAGE = 'usage'; diff --git a/app/init/resources.php b/app/init/resources.php index a9d46a17be..2f43ee008b 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1156,3 +1156,11 @@ App::setResource('resourceToken', function ($project, $dbForProject, $request, A App::setResource('transactionState', function (Database $dbForProject, Authorization $authorization) { return new TransactionState($dbForProject, $authorization); }, ['dbForProject', 'authorization']); + +App::setResource('executionsRetentionCount', function (Document $project, array $plan) { + if ($project->getId() === 'console' || empty($plan)) { + return 0; + } + + return (int) ($plan['executionsRetentionCount'] ?? 100); +}, ['project', 'plan']); diff --git a/app/worker.php b/app/worker.php index ba8bf98568..39f0695bb3 100644 --- a/app/worker.php +++ b/app/worker.php @@ -490,6 +490,14 @@ Server::setResource('getAudit', function (Database $dbForPlatform, callable $get }; }, ['dbForPlatform', 'getProjectDB']); +Server::setResource('executionsRetentionCount', function (Document $project, array $plan) { + if ($project->getId() === 'console' || empty($plan)) { + return 0; + } + + return (int) ($plan['executionsRetentionCount'] ?? 100); +}, ['project', 'plan']); + $pools = $register->get('pools'); $platform = new Appwrite(); $args = $platform->getEnv('argv'); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index 1a265298d3..8c4b68edb6 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Executions; use Ahc\Jwt\JWT; +use Appwrite\Event\Delete as DeleteEvent; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\StatsUsage; @@ -62,7 +63,6 @@ class Create extends Base ->label('scope', 'execution.write') ->label('resourceType', RESOURCE_TYPE_FUNCTIONS) ->label('event', 'functions.[functionId].executions.[executionId].create') - ->label('resourceType', RESOURCE_TYPE_FUNCTIONS) ->label('sdk', new Method( namespace: 'functions', group: 'executions', @@ -101,6 +101,8 @@ class Create extends Base ->inject('executor') ->inject('platform') ->inject('authorization') + ->inject('queueForDeletes') + ->inject('executionsRetentionCount') ->callback($this->action(...)); } @@ -127,6 +129,8 @@ class Create extends Base Executor $executor, array $platform, Authorization $authorization, + DeleteEvent $queueForDeletes, + int $executionsRetentionCount, ) { $async = \strval($async) === 'true' || \strval($async) === '1'; @@ -513,6 +517,16 @@ class Create extends Base } } + /* cleanup */ + if ($executionsRetentionCount > 0) { + $queueForDeletes + ->setProject($project) + ->setResource($function->getSequence()) + ->setResourceType(RESOURCE_TYPE_FUNCTIONS) + ->setType(DELETE_TYPE_EXECUTIONS_LIMIT) + ->trigger(); + } + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($execution, Response::MODEL_EXECUTION); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9687f4f4bb..654b083a98 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -30,6 +30,8 @@ use Utopia\Queue\Message; use Utopia\Storage\Device; use Utopia\System\System; +use function Swoole\Coroutine\batch; + class Deletes extends Action { protected array $selects = ['$sequence', '$id', '$collection', '$permissions', '$updatedAt']; @@ -59,6 +61,7 @@ class Deletes extends Action ->inject('certificates') ->inject('executor') ->inject('executionRetention') + ->inject('executionsRetentionCount') ->inject('auditRetention') ->inject('log') ->inject('getAudit') @@ -83,6 +86,7 @@ class Deletes extends Action CertificatesAdapter $certificates, Executor $executor, string $executionRetention, + int $executionsRetentionCount, string $auditRetention, Log $log, callable $getAudit, @@ -144,6 +148,19 @@ class Deletes extends Action case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); break; + case DELETE_TYPE_EXECUTIONS_LIMIT: + $resourceInternalId = $payload['resource'] ?? null; + $resourceType = $payload['resourceType'] ?? null; + if ($resourceInternalId) { + $this->deleteExecutionsByLimit( + $project, + $getProjectDB, + $executionsRetentionCount, + $resourceInternalId, + $resourceType + ); + } + break; case DELETE_TYPE_AUDIT: if (!$project->isEmpty()) { $this->deleteAuditLogs($project, $getAudit, $auditRetention); @@ -184,7 +201,7 @@ class Deletes extends Action break; case DELETE_TYPE_MAINTENANCE: $this->deleteExpiredTargets($project, $getProjectDB); - $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); + $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention, $executionsRetentionCount); $this->deleteAuditLogs($project, $getAudit, $auditRetention); $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime); $this->deleteExpiredSessions($project, $getProjectDB); @@ -199,16 +216,15 @@ class Deletes extends Action * @param Database $dbForPlatform * @param callable $getProjectDB * @param string $datetime - * @param Document|null $document * @return void * @throws Conflict * @throws Restricted * @throws Structure - * @throws DatabaseException + * @throws DatabaseException|Exception */ private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void { - // Temporarly accepting both 'fra' and 'default' + // Temporarily accepting both 'fra' and 'default' // When all migrated, only use _APP_REGION with 'default' as default value $regions = [System::getEnv('_APP_REGION', 'default')]; if (!in_array('default', $regions)) { @@ -694,14 +710,16 @@ class Deletes extends Action } /** - * @param database $dbForPlatform + * @param Document $project * @param callable $getProjectDB * @param string $datetime + * @param int|null $executionsRetentionCount * @return void - * @throws Exception + * @throws Exception|DatabaseException */ - private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void + private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime, ?int $executionsRetentionCount = 0): void { + /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); // Delete Executions @@ -711,10 +729,85 @@ class Deletes extends Action Query::orderDesc('$createdAt'), Query::orderDesc(), ], $dbForProject); + + /* delete based on custom retention, if any */ + $this->deleteExecutionsByLimit($project, $getProjectDB, $executionsRetentionCount); } /** - * @param Database $dbForPlatform + * @param Document $project + * @param callable $getProjectDB + * @param int|null $executionsRetentionCount + * @param string|null $resourceInternalId + * @param string|null $resourceType + * @return void + * @throws DatabaseException + */ + protected function deleteExecutionsByLimit( + Document $project, + callable $getProjectDB, + ?int $executionsRetentionCount = 0, + ?string $resourceInternalId = null, + ?string $resourceType = null + ): void { + if ($executionsRetentionCount <= 0) { + return; + } + + /** @var Database $dbForProject */ + $dbForProject = $getProjectDB($project); + + /* delete log for a given $resourceInternalId */ + $delete = function (Database $dbForProject, string $resourceInternalId, string $resourceType) use ($executionsRetentionCount) { + // get the execution at position `N+1` + $execution = $dbForProject->findOne('executions', [ + Query::select(['$createdAt']), + Query::equal('resourceInternalId', [$resourceInternalId]), + Query::equal('resourceType', [$resourceType]), + Query::orderDesc('$createdAt'), + Query::offset($executionsRetentionCount), + ]); + + if (!$execution->isEmpty()) { + // delete everything older + $cutoffTime = $execution->getAttribute('$createdAt'); + + $this->deleteByGroup('executions', [ + Query::select([...$this->selects, '$createdAt']), + Query::equal('resourceInternalId', [$resourceInternalId]), + Query::equal('resourceType', [$resourceType]), + Query::lessThan('$createdAt', $cutoffTime), + Query::orderDesc('$createdAt'), + Query::orderDesc(), + ], $dbForProject); + } + }; + + if (!empty($resourceInternalId)) { + // fast path, no need to list anything! + $delete($dbForProject, $resourceInternalId, $resourceType); + } else { + $processResource = function (string $type) use ($dbForProject, $delete, $resourceType) { + $this->listByGroup( + collection: $type, + queries: [Query::select(['$id', '$sequence'])], + database: $dbForProject, + callback: function (Document $resource) use ($dbForProject, $delete, $type) { + $delete($dbForProject, $resource->getSequence(), $type); + } + ); + }; + + /* perform processing in parallel */ + batch([ + fn () => $processResource(RESOURCE_TYPE_SITES), + fn () => $processResource(RESOURCE_TYPE_FUNCTIONS), + ]); + } + } + + /** + * @param Document $project * @param callable $getProjectDB * @return void * @throws Exception|Throwable