From d483fbd43b9d88c5ed5a136337ca7efbf18f9693 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 3 Dec 2025 18:22:46 +0200 Subject: [PATCH] Update composer.lock and refactor ScheduleBase class for improved schedule processing --- composer.lock | 28 ++--- src/Appwrite/Platform/Tasks/ScheduleBase.php | 118 +++++++++++-------- 2 files changed, 84 insertions(+), 62 deletions(-) diff --git a/composer.lock b/composer.lock index 4dccb29a1a..19e9a64c21 100644 --- a/composer.lock +++ b/composer.lock @@ -4952,16 +4952,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.14", + "version": "0.18.16", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00" + "reference": "0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/4f14ec952c6f4006dd0613e55bbf7631814fbc00", - "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f", + "reference": "0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f", "shasum": "" }, "require": { @@ -5004,9 +5004,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.14" + "source": "https://github.com/utopia-php/storage/tree/0.18.16" }, - "time": "2025-10-07T10:21:47+00:00" + "time": "2025-12-03T02:15:45+00:00" }, { "name": "utopia-php/swoole", @@ -6663,16 +6663,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.29", + "version": "9.6.30", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3" + "reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", - "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/b69489b312503bf8fa6d75a76916919d7d2fa6d4", + "reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4", "shasum": "" }, "require": { @@ -6746,7 +6746,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.29" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.30" }, "funding": [ { @@ -6770,7 +6770,7 @@ "type": "tidelift" } ], - "time": "2025-09-24T06:29:11+00:00" + "time": "2025-12-01T07:35:08+00:00" }, { "name": "psr/cache", @@ -8942,7 +8942,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8966,5 +8966,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index e9a0e1d333..641725e722 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -115,36 +115,7 @@ abstract class ScheduleBase extends Action private function collectSchedules(Database $dbForPlatform, callable $getProjectDB, string &$lastSyncUpdate, callable $isResourceBlocked): void { - // If we haven't synced yet, load all active schedules $initialLoad = $lastSyncUpdate === "0"; - - /** - * Extract only necessary attributes to lower memory used. - * - * @return array - * @throws Exception - * @var Document $schedule - */ - $getSchedule = function (Document $schedule) use ($dbForPlatform, $getProjectDB): array { - $project = $dbForPlatform->getDocument('projects', $schedule->getAttribute('projectId')); - - $resource = $getProjectDB($project)->getDocument( - static::getCollectionId(), - $schedule->getAttribute('resourceId') - ); - - return [ - '$sequence' => $schedule->getSequence(), - '$id' => $schedule->getId(), - 'resourceId' => $schedule->getAttribute('resourceId'), - 'schedule' => $schedule->getAttribute('schedule'), - 'active' => $schedule->getAttribute('active'), - 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here - 'resource' => $resource, // TODO: @Meldiron Send only ID to worker to reduce memory usage here - ]; - }; - $loadStart = microtime(true); $time = DateTime::now(); @@ -152,6 +123,9 @@ abstract class ScheduleBase extends Action $sum = $limit; $total = 0; $latestDocument = null; + $collectionId = static::getCollectionId(); + + $schedulesToProcess = []; while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; @@ -160,8 +134,6 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } - // Temporarly accepting both 'fra' and 'default' - // When all migrated, only use _APP_REGION with 'default' as default value $regions = [System::getEnv('_APP_REGION', 'default')]; if (!in_array('default', $regions)) { $regions[] = 'default'; @@ -179,7 +151,6 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate); } - $collectionId = static::getCollectionId(); $schedules = $dbForPlatform->find('schedules', $paginationQueries); $sum = count($schedules); $total += $sum; @@ -189,32 +160,83 @@ abstract class ScheduleBase extends Action $updated = strtotime($existing['resourceUpdatedAt'] ?? '0') !== strtotime($schedule['resourceUpdatedAt'] ?? '0'); if ($existing === null || $updated) { - try { - $candidate = $getSchedule($schedule); - } catch (\Throwable $th) { - Console::error("Failed to load schedule for project {$schedule['projectId']} {$collectionId} {$schedule['resourceId']}"); - Console::error($th->getMessage()); - continue; - } - - if (!$candidate['active']) { + // Early filtering: skip if not active (only for updates, initial load already filters) + if (!$initialLoad && !$schedule->getAttribute('active', true)) { unset($this->schedules[$schedule->getSequence()]); continue; } - if ($isResourceBlocked($candidate['project'], $collectionId, $candidate['resourceId'])) { - unset($this->schedules[$schedule->getSequence()]); - continue; - } - - Console::info("Updating: {$schedule['resourceType']}::{$schedule['resourceId']}"); - $this->schedules[$schedule->getSequence()] = $candidate; + $schedulesToProcess[] = $schedule; } } $latestDocument = \end($schedules); } + if (empty($schedulesToProcess)) { + $lastSyncUpdate = $time; + $duration = microtime(true) - $loadStart; + $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + $this->collectSchedulesTelemetryCount->record($total, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + Console::success("{$total} resources were loaded in " . $duration . " seconds"); + return; + } + + // Cache projects to avoid fetching the same project multiple times + $projectsCache = []; + + foreach ($schedulesToProcess as $schedule) { + $projectId = $schedule->getAttribute('projectId'); + + // Fetch project if not already cached + if (!isset($projectsCache[$projectId])) { + try { + $projectsCache[$projectId] = $dbForPlatform->getDocument('projects', $projectId); + } catch (\Throwable $th) { + Console::error("Failed to load project {$projectId}: " . $th->getMessage()); + continue; + } + } + + $project = $projectsCache[$projectId]; + if ($project === null) { + continue; + } + + try { + $dbForProject = $getProjectDB($project); + $resourceId = $schedule->getAttribute('resourceId'); + $resource = $dbForProject->getDocument($collectionId, $resourceId); + + $candidate = [ + '$sequence' => $schedule->getSequence(), + '$id' => $schedule->getId(), + 'resourceId' => $resourceId, + 'schedule' => $schedule->getAttribute('schedule'), + 'active' => $schedule->getAttribute('active'), + 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), + 'project' => $project, + 'resource' => $resource, + ]; + + // Early filtering checks + if (!$candidate['active']) { + unset($this->schedules[$schedule->getSequence()]); + continue; + } + + if ($isResourceBlocked($candidate['project'], $collectionId, $candidate['resourceId'])) { + unset($this->schedules[$schedule->getSequence()]); + continue; + } + + Console::info("loading: project:: " . $candidate['project']->getId() . " " . static::getSupportedResource() . "::{$candidate['resourceId']}"); + $this->schedules[$schedule->getSequence()] = $candidate; + } catch (\Throwable $th) { + Console::error("Failed to load schedule for project {$project->getId()} {$collectionId} {$schedule->getAttribute('resourceId')}: " . $th->getMessage()); + } + } + $lastSyncUpdate = $time; $duration = microtime(true) - $loadStart; $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]);