From e8b11b2d14c25c8486a87ffd2196846b2a15feaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 19 Mar 2023 10:43:57 +0100 Subject: [PATCH 1/5] Fix scheduler --- app/config/collections.php | 11 ---- app/controllers/api/functions.php | 71 ++++++++++++-------------- app/workers/builds.php | 4 +- src/Appwrite/Migration/Version/V17.php | 11 ++++ 4 files changed, 45 insertions(+), 52 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index afde813fa1..4c71a404a0 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2278,17 +2278,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('scheduleUpdatedAt'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], [ '$id' => ID::custom('timeout'), 'type' => Database::VAR_INTEGER, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 6fe7c67e69..67674e3bb5 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -84,7 +84,6 @@ App::post('/v1/functions') 'deployment' => '', 'events' => $events, 'schedule' => $schedule, - 'scheduleUpdatedAt' => DateTime::now(), 'timeout' => $timeout, 'search' => implode(' ', [$functionId, $name, $runtime]) ])); @@ -463,7 +462,6 @@ App::put('/v1/functions/:functionId') 'name' => $name, 'events' => $events, 'schedule' => $schedule, - 'scheduleUpdatedAt' => DateTime::now(), 'timeout' => $timeout, 'enabled' => $enabled, 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), @@ -471,18 +469,11 @@ App::put('/v1/functions/:functionId') $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - /** - * In case we want to clear the schedule - */ - if (!empty($function->getAttribute('deployment'))) { - $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); - } - $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $eventsInstance->setParam('functionId', $function->getId()); @@ -538,15 +529,10 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') ]))); $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - - $active = !empty($function->getAttribute('schedule')); - - if ($active) { - $schedule->setAttribute('resourceUpdatedAt', datetime::now()); - } - - $schedule->setAttribute('active', $active); - + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); $events @@ -786,22 +772,6 @@ App::post('/v1/functions/:functionId/deployments') } } - /** - * TODO Should we update also the function collection with the scheduleUpdatedAt attr? - */ - - $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - - $active = !empty($function->getAttribute('schedule')); - - if ($active) { - $schedule->setAttribute('resourceUpdatedAt', datetime::now()); - } - - $schedule->setAttribute('active', $active); - - Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); - $metadata = null; $events @@ -1381,7 +1351,8 @@ App::post('/v1/functions/:functionId/variables') ->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', false) ->inject('response') ->inject('dbForProject') - ->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1410,6 +1381,13 @@ App::post('/v1/functions/:functionId/variables') throw new Exception(Exception::VARIABLE_ALREADY_EXISTS); } + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $dbForProject->deleteCachedDocument('functions', $function->getId()); $response @@ -1497,7 +1475,8 @@ App::put('/v1/functions/:functionId/variables/:variableId') ->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); @@ -1526,6 +1505,13 @@ App::put('/v1/functions/:functionId/variables/:variableId') throw new Exception(Exception::VARIABLE_ALREADY_EXISTS); } + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $dbForProject->deleteCachedDocument('functions', $function->getId()); $response->dynamic($variable, Response::MODEL_VARIABLE); @@ -1547,7 +1533,8 @@ App::delete('/v1/functions/:functionId/variables/:variableId') ->param('variableId', '', new UID(), 'Variable unique ID.', false) ->inject('response') ->inject('dbForProject') - ->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject) { + ->inject('dbForConsole') + ->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1564,6 +1551,14 @@ App::delete('/v1/functions/:functionId/variables/:variableId') } $dbForProject->deleteDocument('variables', $variable->getId()); + + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + $dbForProject->deleteCachedDocument('functions', $function->getId()); $response->noContent(); diff --git a/app/workers/builds.php b/app/workers/builds.php index d26f07ab75..9dfd69b872 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -202,8 +202,6 @@ class BuildsV1 extends Worker Console::success("Build id: $buildId created"); - $function->setAttribute('scheduleUpdatedAt', DateTime::now()); - /** Set auto deploy */ if ($deployment->getAttribute('activate') === true) { $function->setAttribute('deployment', $deployment->getId()); @@ -213,7 +211,7 @@ class BuildsV1 extends Worker /** Update function schedule */ $dbForConsole = $this->getConsoleDB(); $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); + $schedule->setAttribute('resourceUpdatedAt', DateTime::now()); $schedule ->setAttribute('schedule', $function->getAttribute('schedule')) diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index 66a02662d1..5ee897ae19 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -58,6 +58,17 @@ class V17 extends Migration } break; + case 'functions': + try { + /** + * Delete 'scheduleUpdatedAt' attribute + */ + $this->projectDB->deleteAttribute($id, 'scheduleUpdatedAt'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'scheduleUpdatedAt' from {$id}: {$th->getMessage()}"); + } + break; default: break; From cf39092c98e16ae67335003777e213fb82f3ffbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 22 Mar 2023 19:52:43 +0100 Subject: [PATCH 2/5] Add patch task for scheduleUpdatedAt --- src/Appwrite/Migration/Version/V17.php | 11 --- src/Appwrite/Platform/Services/Tasks.php | 2 + .../PatchDeleteScheduleUpdatedAtAttribute.php | 70 +++++++++++++++++++ 3 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index 5ee897ae19..66a02662d1 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -58,17 +58,6 @@ class V17 extends Migration } break; - case 'functions': - try { - /** - * Delete 'scheduleUpdatedAt' attribute - */ - $this->projectDB->deleteAttribute($id, 'scheduleUpdatedAt'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'scheduleUpdatedAt' from {$id}: {$th->getMessage()}"); - } - break; default: break; diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 00cf3f89c5..a0c5d3a547 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -13,6 +13,7 @@ use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Hamster; +use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; @@ -33,6 +34,7 @@ class Tasks extends Service ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules()) + ->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php b/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php new file mode 100644 index 0000000000..4ee213303c --- /dev/null +++ b/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php @@ -0,0 +1,70 @@ +desc('Ensure function collections do not have scheduleUpdatedAt attribute') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + } + + /** + * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. + */ + public function action(Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + Console::title('PatchDeleteScheduleUpdatedAtAttribute V1'); + Console::success(APP_NAME . ' PatchDeleteScheduleUpdatedAtAttribute v1 has started'); + + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); + + if (count($projects) === 0) { + break; + } + + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")"); + $dbForProject = $getProjectDB($project); + + try { + /** + * Delete 'scheduleUpdatedAt' attribute + */ + $dbForProject->deleteAttribute('functions', 'scheduleUpdatedAt'); + $dbForProject->deleteCachedCollection('functions'); + Console::success("'scheduleUpdatedAt' deleted."); + } catch (\Throwable $th) { + Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}"); + } + } + + $projectCursor = $projects[array_key_last($projects)]; + } + } +} From b558a284f3e6a224c0ed5273ea172875a9db3802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 22 Mar 2023 20:15:01 +0100 Subject: [PATCH 3/5] Add bin files --- Dockerfile | 1 + bin/patch-delete-schedule-updated-at-attribute | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 bin/patch-delete-schedule-updated-at-attribute diff --git a/Dockerfile b/Dockerfile index abe9b7128f..a29afa2baf 100755 --- a/Dockerfile +++ b/Dockerfile @@ -117,6 +117,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/patch-create-missing-schedules && \ + chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ diff --git a/bin/patch-delete-schedule-updated-at-attribute b/bin/patch-delete-schedule-updated-at-attribute new file mode 100644 index 0000000000..3e28289cbe --- /dev/null +++ b/bin/patch-delete-schedule-updated-at-attribute @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@ \ No newline at end of file From 567cb01b9940948575abb08a1654b92cc61fe6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 22 Mar 2023 20:35:04 +0100 Subject: [PATCH 4/5] add reclaim logic --- .../Tasks/PatchDeleteScheduleUpdatedAtAttribute.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php b/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php index 4ee213303c..95a7c4ffe1 100644 --- a/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php +++ b/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php @@ -7,6 +7,7 @@ use Utopia\CLI\Console; use Utopia\Database\Query; use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; +use Utopia\Pools\Group; class PatchDeleteScheduleUpdatedAtAttribute extends Action { @@ -19,15 +20,16 @@ class PatchDeleteScheduleUpdatedAtAttribute extends Action { $this ->desc('Ensure function collections do not have scheduleUpdatedAt attribute') + ->inject('pools') ->inject('dbForConsole') ->inject('getProjectDB') - ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + ->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB)); } /** * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. */ - public function action(Database $dbForConsole, callable $getProjectDB): void + public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void { Authorization::disable(); Authorization::setDefaultStatus(false); @@ -62,6 +64,8 @@ class PatchDeleteScheduleUpdatedAtAttribute extends Action } catch (\Throwable $th) { Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}"); } + + $pools->reclaim(); } $projectCursor = $projects[array_key_last($projects)]; From bd620cc9240015a9182ca648bee17676ff0b6bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 22 Mar 2023 20:39:56 +0100 Subject: [PATCH 5/5] Remove old patch bin file --- Dockerfile | 1 - bin/patch-create-missing-schedules | 3 --- 2 files changed, 4 deletions(-) delete mode 100644 bin/patch-create-missing-schedules diff --git a/Dockerfile b/Dockerfile index a29afa2baf..eaf8b73e1b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -116,7 +116,6 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ - chmod +x /usr/local/bin/patch-create-missing-schedules && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ diff --git a/bin/patch-create-missing-schedules b/bin/patch-create-missing-schedules deleted file mode 100644 index e38d3e9a6f..0000000000 --- a/bin/patch-create-missing-schedules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-create-missing-schedules $@ \ No newline at end of file