From d483fbd43b9d88c5ed5a136337ca7efbf18f9693 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 3 Dec 2025 18:22:46 +0200 Subject: [PATCH 1/6] Update composer.lock and refactor ScheduleBase class for improved schedule processing --- composer.lock | 28 ++--- src/Appwrite/Platform/Tasks/ScheduleBase.php | 118 +++++++++++-------- 2 files changed, 84 insertions(+), 62 deletions(-) diff --git a/composer.lock b/composer.lock index 4dccb29a1a..19e9a64c21 100644 --- a/composer.lock +++ b/composer.lock @@ -4952,16 +4952,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.14", + "version": "0.18.16", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00" + "reference": "0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/4f14ec952c6f4006dd0613e55bbf7631814fbc00", - "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f", + "reference": "0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f", "shasum": "" }, "require": { @@ -5004,9 +5004,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.14" + "source": "https://github.com/utopia-php/storage/tree/0.18.16" }, - "time": "2025-10-07T10:21:47+00:00" + "time": "2025-12-03T02:15:45+00:00" }, { "name": "utopia-php/swoole", @@ -6663,16 +6663,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.29", + "version": "9.6.30", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3" + "reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", - "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/b69489b312503bf8fa6d75a76916919d7d2fa6d4", + "reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4", "shasum": "" }, "require": { @@ -6746,7 +6746,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.29" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.30" }, "funding": [ { @@ -6770,7 +6770,7 @@ "type": "tidelift" } ], - "time": "2025-09-24T06:29:11+00:00" + "time": "2025-12-01T07:35:08+00:00" }, { "name": "psr/cache", @@ -8942,7 +8942,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8966,5 +8966,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index e9a0e1d333..641725e722 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -115,36 +115,7 @@ abstract class ScheduleBase extends Action private function collectSchedules(Database $dbForPlatform, callable $getProjectDB, string &$lastSyncUpdate, callable $isResourceBlocked): void { - // If we haven't synced yet, load all active schedules $initialLoad = $lastSyncUpdate === "0"; - - /** - * Extract only necessary attributes to lower memory used. - * - * @return array - * @throws Exception - * @var Document $schedule - */ - $getSchedule = function (Document $schedule) use ($dbForPlatform, $getProjectDB): array { - $project = $dbForPlatform->getDocument('projects', $schedule->getAttribute('projectId')); - - $resource = $getProjectDB($project)->getDocument( - static::getCollectionId(), - $schedule->getAttribute('resourceId') - ); - - return [ - '$sequence' => $schedule->getSequence(), - '$id' => $schedule->getId(), - 'resourceId' => $schedule->getAttribute('resourceId'), - 'schedule' => $schedule->getAttribute('schedule'), - 'active' => $schedule->getAttribute('active'), - 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here - 'resource' => $resource, // TODO: @Meldiron Send only ID to worker to reduce memory usage here - ]; - }; - $loadStart = microtime(true); $time = DateTime::now(); @@ -152,6 +123,9 @@ abstract class ScheduleBase extends Action $sum = $limit; $total = 0; $latestDocument = null; + $collectionId = static::getCollectionId(); + + $schedulesToProcess = []; while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; @@ -160,8 +134,6 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } - // Temporarly accepting both 'fra' and 'default' - // When all migrated, only use _APP_REGION with 'default' as default value $regions = [System::getEnv('_APP_REGION', 'default')]; if (!in_array('default', $regions)) { $regions[] = 'default'; @@ -179,7 +151,6 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate); } - $collectionId = static::getCollectionId(); $schedules = $dbForPlatform->find('schedules', $paginationQueries); $sum = count($schedules); $total += $sum; @@ -189,32 +160,83 @@ abstract class ScheduleBase extends Action $updated = strtotime($existing['resourceUpdatedAt'] ?? '0') !== strtotime($schedule['resourceUpdatedAt'] ?? '0'); if ($existing === null || $updated) { - try { - $candidate = $getSchedule($schedule); - } catch (\Throwable $th) { - Console::error("Failed to load schedule for project {$schedule['projectId']} {$collectionId} {$schedule['resourceId']}"); - Console::error($th->getMessage()); - continue; - } - - if (!$candidate['active']) { + // Early filtering: skip if not active (only for updates, initial load already filters) + if (!$initialLoad && !$schedule->getAttribute('active', true)) { unset($this->schedules[$schedule->getSequence()]); continue; } - if ($isResourceBlocked($candidate['project'], $collectionId, $candidate['resourceId'])) { - unset($this->schedules[$schedule->getSequence()]); - continue; - } - - Console::info("Updating: {$schedule['resourceType']}::{$schedule['resourceId']}"); - $this->schedules[$schedule->getSequence()] = $candidate; + $schedulesToProcess[] = $schedule; } } $latestDocument = \end($schedules); } + if (empty($schedulesToProcess)) { + $lastSyncUpdate = $time; + $duration = microtime(true) - $loadStart; + $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + $this->collectSchedulesTelemetryCount->record($total, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + Console::success("{$total} resources were loaded in " . $duration . " seconds"); + return; + } + + // Cache projects to avoid fetching the same project multiple times + $projectsCache = []; + + foreach ($schedulesToProcess as $schedule) { + $projectId = $schedule->getAttribute('projectId'); + + // Fetch project if not already cached + if (!isset($projectsCache[$projectId])) { + try { + $projectsCache[$projectId] = $dbForPlatform->getDocument('projects', $projectId); + } catch (\Throwable $th) { + Console::error("Failed to load project {$projectId}: " . $th->getMessage()); + continue; + } + } + + $project = $projectsCache[$projectId]; + if ($project === null) { + continue; + } + + try { + $dbForProject = $getProjectDB($project); + $resourceId = $schedule->getAttribute('resourceId'); + $resource = $dbForProject->getDocument($collectionId, $resourceId); + + $candidate = [ + '$sequence' => $schedule->getSequence(), + '$id' => $schedule->getId(), + 'resourceId' => $resourceId, + 'schedule' => $schedule->getAttribute('schedule'), + 'active' => $schedule->getAttribute('active'), + 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), + 'project' => $project, + 'resource' => $resource, + ]; + + // Early filtering checks + if (!$candidate['active']) { + unset($this->schedules[$schedule->getSequence()]); + continue; + } + + if ($isResourceBlocked($candidate['project'], $collectionId, $candidate['resourceId'])) { + unset($this->schedules[$schedule->getSequence()]); + continue; + } + + Console::info("loading: project:: " . $candidate['project']->getId() . " " . static::getSupportedResource() . "::{$candidate['resourceId']}"); + $this->schedules[$schedule->getSequence()] = $candidate; + } catch (\Throwable $th) { + Console::error("Failed to load schedule for project {$project->getId()} {$collectionId} {$schedule->getAttribute('resourceId')}: " . $th->getMessage()); + } + } + $lastSyncUpdate = $time; $duration = microtime(true) - $loadStart; $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); From 7462b7ae170fce0a1bb6e6591ea107f9778e2b03 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 7 Dec 2025 16:57:30 +0200 Subject: [PATCH 2/6] Refactor ScheduleBase class to enhance schedule loading and project management. --- src/Appwrite/Platform/Tasks/ScheduleBase.php | 163 ++++++++++++------- 1 file changed, 102 insertions(+), 61 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 641725e722..bd5ee6bfa0 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -123,9 +123,7 @@ abstract class ScheduleBase extends Action $sum = $limit; $total = 0; $latestDocument = null; - $collectionId = static::getCollectionId(); - - $schedulesToProcess = []; + $updatedProjectIds = []; // Track project IDs from updated/new schedules while ($sum === $limit) { $paginationQueries = [Query::limit($limit)]; @@ -134,6 +132,8 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value $regions = [System::getEnv('_APP_REGION', 'default')]; if (!in_array('default', $regions)) { $regions[] = 'default'; @@ -151,97 +151,138 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate); } + $collectionId = static::getCollectionId(); $schedules = $dbForPlatform->find('schedules', $paginationQueries); $sum = count($schedules); $total += $sum; foreach ($schedules as $schedule) { $existing = $this->schedules[$schedule->getSequence()] ?? null; - $updated = strtotime($existing['resourceUpdatedAt'] ?? '0') !== strtotime($schedule['resourceUpdatedAt'] ?? '0'); - + $updated = strtotime($existing['resourceUpdatedAt'] ?? '0') !== strtotime($schedule->getAttribute('resourceUpdatedAt') ?? '0'); + if ($existing === null || $updated) { - // Early filtering: skip if not active (only for updates, initial load already filters) - if (!$initialLoad && !$schedule->getAttribute('active', true)) { + try { + $candidate = [ + '$sequence' => $schedule->getSequence(), + '$id' => $schedule->getId(), + 'projectId' => $schedule->getAttribute('projectId'), + 'resourceId' => $schedule->getAttribute('resourceId'), + 'resourceType' => $schedule->getAttribute('resourceType'), + 'schedule' => $schedule->getAttribute('schedule'), + 'active' => $schedule->getAttribute('active'), + 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), + ]; + } catch (\Throwable $th) { + Console::error("Failed to load schedule for project {$schedule->getAttribute('projectId')} {$collectionId} {$schedule->getAttribute('resourceId')}"); + Console::error($th->getMessage()); + continue; + } + // In case the resource is not active (deleted). + if (!$candidate['active']) { + Console::error("Resource is not active: {$candidate['resourceType']}::{$candidate['resourceId']}"); unset($this->schedules[$schedule->getSequence()]); continue; } - $schedulesToProcess[] = $schedule; + Console::info("Updating: {$candidate['resourceType']}::{$candidate['resourceId']}"); + $this->schedules[$schedule->getSequence()] = $candidate; + + // Track projectId for updated/new schedules + $updatedProjectIds[] = $candidate['projectId']; } } $latestDocument = \end($schedules); } - - if (empty($schedulesToProcess)) { - $lastSyncUpdate = $time; - $duration = microtime(true) - $loadStart; - $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); - $this->collectSchedulesTelemetryCount->record($total, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); - Console::success("{$total} resources were loaded in " . $duration . " seconds"); + if (empty($this->schedules)) { + Console::success("No resources found"); return; } - // Cache projects to avoid fetching the same project multiple times - $projectsCache = []; + // On initial load: load all projects from all schedules + if ($initialLoad) { + $projectIds = array_unique(array_map(fn($schedule) => $schedule['projectId'], $this->schedules)); + } else { + // Only load projects for updated/new schedules + $projectIds = array_unique($updatedProjectIds); + } + + // Build existing project map from schedules that already have projects loaded + $map = []; + foreach ($this->schedules as $schedule) { + if (isset($schedule['project'])) { + $map[$schedule['projectId']] = $schedule['project']; + } + } + + // Only load projects that we don't already have in memory + $projectIdsToLoad = array_filter($projectIds, fn($projectId) => !isset($map[$projectId])); + + if (!empty($projectIdsToLoad)) { + $projectIdsToLoad = array_values($projectIdsToLoad); + $batchSize = 10_000; + $batches = array_chunk($projectIdsToLoad, $batchSize); + $projectsLoadStart = microtime(true); + + foreach ($batches as $batch) { + $documents = $dbForPlatform->find('projects', [ + Query::equal('$id', $batch), + Query::limit(count($batch)), + ]); - foreach ($schedulesToProcess as $schedule) { - $projectId = $schedule->getAttribute('projectId'); - - // Fetch project if not already cached - if (!isset($projectsCache[$projectId])) { - try { - $projectsCache[$projectId] = $dbForPlatform->getDocument('projects', $projectId); - } catch (\Throwable $th) { - Console::error("Failed to load project {$projectId}: " . $th->getMessage()); - continue; + foreach ($documents as $document) { + $map[$document->getId()] = $document; } } - $project = $projectsCache[$projectId]; - if ($project === null) { + $projectsLoadDuration = microtime(true) - $projectsLoadStart; + Console::success("Projects map loaded in " . $projectsLoadDuration . " seconds with " . count($projectIdsToLoad) . " new projects (total: " . count($map) . " projects)"); + } else { + Console::success("No new projects to load (using " . count($map) . " cached projects)"); + } + + foreach ($this->schedules as $sequence => $schedule) { + $project = $map[$schedule['projectId']]; + + // In case the resource is blocked. + if ($isResourceBlocked($project, $collectionId, $schedule['resourceId'])) { + Console::error("Resource blocked: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); + unset($this->schedules[$sequence]); continue; } - try { - $dbForProject = $getProjectDB($project); - $resourceId = $schedule->getAttribute('resourceId'); - $resource = $dbForProject->getDocument($collectionId, $resourceId); - - $candidate = [ - '$sequence' => $schedule->getSequence(), - '$id' => $schedule->getId(), - 'resourceId' => $resourceId, - 'schedule' => $schedule->getAttribute('schedule'), - 'active' => $schedule->getAttribute('active'), - 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'project' => $project, - 'resource' => $resource, - ]; - - // Early filtering checks - if (!$candidate['active']) { - unset($this->schedules[$schedule->getSequence()]); - continue; - } - - if ($isResourceBlocked($candidate['project'], $collectionId, $candidate['resourceId'])) { - unset($this->schedules[$schedule->getSequence()]); - continue; - } - - Console::info("loading: project:: " . $candidate['project']->getId() . " " . static::getSupportedResource() . "::{$candidate['resourceId']}"); - $this->schedules[$schedule->getSequence()] = $candidate; - } catch (\Throwable $th) { - Console::error("Failed to load schedule for project {$project->getId()} {$collectionId} {$schedule->getAttribute('resourceId')}: " . $th->getMessage()); + if (empty($project)) { + Console::error("Project not found: projetId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); + unset($this->schedules[$sequence]); + continue; } + + $this->schedules[$sequence]['project'] = $project; + + // In case the resource is not found (project deleted). + try { + $resource = $getProjectDB($project)->getDocument(static::getCollectionId(), $schedule['resourceId']); + } catch (\Throwable $th) { + Console::error("Failed to load resource: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); + Console::error($th->getMessage()); + unset($this->schedules[$sequence]); + continue; + } + + if (empty($resource)) { + Console::error("Resource not found: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); + unset($this->schedules[$sequence]); + continue; + } + + $this->schedules[$sequence]['resource'] = $resource; } $lastSyncUpdate = $time; $duration = microtime(true) - $loadStart; $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); $this->collectSchedulesTelemetryCount->record($total, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); - Console::success("{$total} resources were loaded in " . $duration . " seconds"); + Console::success("Timer loaded {$total} " . static::getName() . " in " . $duration . " seconds"); } protected function recordEnqueueDelay(\DateTime $expectedExecutionSchedule): void From 8ed4fa6ff11c55ebd1d4aac03d50186a1187c9a5 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 7 Dec 2025 17:03:18 +0200 Subject: [PATCH 3/6] composer --- composer.lock | 48 ++++++++++---------- src/Appwrite/Platform/Tasks/ScheduleBase.php | 25 +++++----- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/composer.lock b/composer.lock index 19e9a64c21..e1fd7fcf17 100644 --- a/composer.lock +++ b/composer.lock @@ -756,16 +756,16 @@ }, { "name": "google/protobuf", - "version": "v4.33.1", + "version": "v4.33.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12" + "reference": "fbd96b7bf1343f4b0d8fb358526c7ba4d72f1318" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/0cd73ccf0cd26c3e72299cce1ea6144091a57e12", - "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/fbd96b7bf1343f4b0d8fb358526c7ba4d72f1318", + "reference": "fbd96b7bf1343f4b0d8fb358526c7ba4d72f1318", "shasum": "" }, "require": { @@ -794,9 +794,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.2" }, - "time": "2025-11-12T21:58:05+00:00" + "time": "2025-12-05T22:12:22+00:00" }, { "name": "league/csv", @@ -4513,16 +4513,16 @@ }, { "name": "utopia-php/migration", - "version": "1.3.6", + "version": "1.3.7", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "4abe70cc242bbffebfa377e4126ee2a4a1c9ef7e" + "reference": "409983bc2a9cf53a8a3dc6c23d8b1dee1950b499" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/4abe70cc242bbffebfa377e4126ee2a4a1c9ef7e", - "reference": "4abe70cc242bbffebfa377e4126ee2a4a1c9ef7e", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/409983bc2a9cf53a8a3dc6c23d8b1dee1950b499", + "reference": "409983bc2a9cf53a8a3dc6c23d8b1dee1950b499", "shasum": "" }, "require": { @@ -4562,9 +4562,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.3.6" + "source": "https://github.com/utopia-php/migration/tree/1.3.7" }, - "time": "2025-11-25T11:36:57+00:00" + "time": "2025-12-05T05:02:31+00:00" }, { "name": "utopia-php/mongo", @@ -5960,16 +5960,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.6.2", + "version": "v5.7.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "3a454ca033b9e06b63282ce19562e892747449bb" + "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/3a454ca033b9e06b63282ce19562e892747449bb", - "reference": "3a454ca033b9e06b63282ce19562e892747449bb", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/dca41cd15c2ac9d055ad70dbfd011130757d1f82", + "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82", "shasum": "" }, "require": { @@ -6012,9 +6012,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.6.2" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.7.0" }, - "time": "2025-10-21T19:32:17+00:00" + "time": "2025-12-06T11:56:16+00:00" }, { "name": "phar-io/manifest", @@ -6663,16 +6663,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.30", + "version": "9.6.31", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4" + "reference": "945d0b7f346a084ce5549e95289962972c4272e5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/b69489b312503bf8fa6d75a76916919d7d2fa6d4", - "reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/945d0b7f346a084ce5549e95289962972c4272e5", + "reference": "945d0b7f346a084ce5549e95289962972c4272e5", "shasum": "" }, "require": { @@ -6746,7 +6746,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.30" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.31" }, "funding": [ { @@ -6770,7 +6770,7 @@ "type": "tidelift" } ], - "time": "2025-12-01T07:35:08+00:00" + "time": "2025-12-06T07:45:52+00:00" }, { "name": "psr/cache", diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index bd5ee6bfa0..f0939e10c5 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -7,7 +7,6 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; @@ -159,7 +158,7 @@ abstract class ScheduleBase extends Action foreach ($schedules as $schedule) { $existing = $this->schedules[$schedule->getSequence()] ?? null; $updated = strtotime($existing['resourceUpdatedAt'] ?? '0') !== strtotime($schedule->getAttribute('resourceUpdatedAt') ?? '0'); - + if ($existing === null || $updated) { try { $candidate = [ @@ -186,7 +185,7 @@ abstract class ScheduleBase extends Action Console::info("Updating: {$candidate['resourceType']}::{$candidate['resourceId']}"); $this->schedules[$schedule->getSequence()] = $candidate; - + // Track projectId for updated/new schedules $updatedProjectIds[] = $candidate['projectId']; } @@ -201,12 +200,12 @@ abstract class ScheduleBase extends Action // On initial load: load all projects from all schedules if ($initialLoad) { - $projectIds = array_unique(array_map(fn($schedule) => $schedule['projectId'], $this->schedules)); + $projectIds = array_unique(array_map(fn ($schedule) => $schedule['projectId'], $this->schedules)); } else { // Only load projects for updated/new schedules $projectIds = array_unique($updatedProjectIds); } - + // Build existing project map from schedules that already have projects loaded $map = []; foreach ($this->schedules as $schedule) { @@ -214,16 +213,16 @@ abstract class ScheduleBase extends Action $map[$schedule['projectId']] = $schedule['project']; } } - + // Only load projects that we don't already have in memory - $projectIdsToLoad = array_filter($projectIds, fn($projectId) => !isset($map[$projectId])); - + $projectIdsToLoad = array_filter($projectIds, fn ($projectId) => !isset($map[$projectId])); + if (!empty($projectIdsToLoad)) { $projectIdsToLoad = array_values($projectIdsToLoad); $batchSize = 10_000; $batches = array_chunk($projectIdsToLoad, $batchSize); $projectsLoadStart = microtime(true); - + foreach ($batches as $batch) { $documents = $dbForPlatform->find('projects', [ Query::equal('$id', $batch), @@ -240,10 +239,10 @@ abstract class ScheduleBase extends Action } else { Console::success("No new projects to load (using " . count($map) . " cached projects)"); } - + foreach ($this->schedules as $sequence => $schedule) { $project = $map[$schedule['projectId']]; - + // In case the resource is blocked. if ($isResourceBlocked($project, $collectionId, $schedule['resourceId'])) { Console::error("Resource blocked: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); @@ -256,7 +255,7 @@ abstract class ScheduleBase extends Action unset($this->schedules[$sequence]); continue; } - + $this->schedules[$sequence]['project'] = $project; // In case the resource is not found (project deleted). @@ -268,7 +267,7 @@ abstract class ScheduleBase extends Action unset($this->schedules[$sequence]); continue; } - + if (empty($resource)) { Console::error("Resource not found: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); unset($this->schedules[$sequence]); From a241eab792bfc2b1438227edf85c51227adf616d Mon Sep 17 00:00:00 2001 From: Shimon Newman Date: Sun, 7 Dec 2025 18:26:14 +0200 Subject: [PATCH 4/6] Update src/Appwrite/Platform/Tasks/ScheduleBase.php Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/Appwrite/Platform/Tasks/ScheduleBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index f0939e10c5..74c91c193d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -251,7 +251,7 @@ abstract class ScheduleBase extends Action } if (empty($project)) { - Console::error("Project not found: projetId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); + Console::error("Project not found: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); unset($this->schedules[$sequence]); continue; } From 6156b97ea03c18654ada3e3c1bf2d6a44dbcb4de Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 8 Dec 2025 17:49:10 +0200 Subject: [PATCH 5/6] Refactor error handling in ScheduleBase class to improve project and resource validation. --- composer.lock | 95 ++++++++++---------- src/Appwrite/Platform/Tasks/ScheduleBase.php | 14 +-- 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/composer.lock b/composer.lock index e1fd7fcf17..8a85fe3229 100644 --- a/composer.lock +++ b/composer.lock @@ -2673,16 +2673,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.0", + "version": "v7.4.1", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ee5e0e0139ab506f6063a230e631bed677c650a4" + "reference": "26cc224ea7103dda90e9694d9e139a389092d007" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ee5e0e0139ab506f6063a230e631bed677c650a4", - "reference": "ee5e0e0139ab506f6063a230e631bed677c650a4", + "url": "https://api.github.com/repos/symfony/http-client/zipball/26cc224ea7103dda90e9694d9e139a389092d007", + "reference": "26cc224ea7103dda90e9694d9e139a389092d007", "shasum": "" }, "require": { @@ -2750,7 +2750,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.0" + "source": "https://github.com/symfony/http-client/tree/v7.4.1" }, "funding": [ { @@ -2770,7 +2770,7 @@ "type": "tidelift" } ], - "time": "2025-11-20T12:32:50+00:00" + "time": "2025-12-04T21:12:57+00:00" }, { "name": "symfony/http-client-contracts", @@ -4264,29 +4264,29 @@ }, { "name": "utopia-php/framework", - "version": "0.33.33", + "version": "0.33.34", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "838e3a28276e73187bc34a314f014096dc92191b" + "reference": "76def92594c32504ec80eaacdb60ff8fad73c856" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/838e3a28276e73187bc34a314f014096dc92191b", - "reference": "838e3a28276e73187bc34a314f014096dc92191b", + "url": "https://api.github.com/repos/utopia-php/http/zipball/76def92594c32504ec80eaacdb60ff8fad73c856", + "reference": "76def92594c32504ec80eaacdb60ff8fad73c856", "shasum": "" }, "require": { - "php": ">=8.1", + "php": ">=8.3", "utopia-php/compression": "0.1.*", "utopia-php/telemetry": "0.1.*", "utopia-php/validators": "0.1.*" }, "require-dev": { - "laravel/pint": "^1.2", - "phpbench/phpbench": "^1.2", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^9.5.25" + "laravel/pint": "1.*", + "phpbench/phpbench": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "9.*" }, "type": "library", "autoload": { @@ -4306,9 +4306,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.33" + "source": "https://github.com/utopia-php/http/tree/0.33.34" }, - "time": "2025-11-25T10:21:13+00:00" + "time": "2025-12-08T07:55:31+00:00" }, { "name": "utopia-php/image", @@ -4513,16 +4513,16 @@ }, { "name": "utopia-php/migration", - "version": "1.3.7", + "version": "1.3.9", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "409983bc2a9cf53a8a3dc6c23d8b1dee1950b499" + "reference": "c55ec67c74663190cda10fd79297422147be7e85" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/409983bc2a9cf53a8a3dc6c23d8b1dee1950b499", - "reference": "409983bc2a9cf53a8a3dc6c23d8b1dee1950b499", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/c55ec67c74663190cda10fd79297422147be7e85", + "reference": "c55ec67c74663190cda10fd79297422147be7e85", "shasum": "" }, "require": { @@ -4562,9 +4562,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.3.7" + "source": "https://github.com/utopia-php/migration/tree/1.3.9" }, - "time": "2025-12-05T05:02:31+00:00" + "time": "2025-12-08T08:45:09+00:00" }, { "name": "utopia-php/mongo", @@ -4679,16 +4679,16 @@ }, { "name": "utopia-php/platform", - "version": "0.7.12", + "version": "0.7.13", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "04255de21db75e90b170040f4d1b457ba721e7a5" + "reference": "77a863a920122e2c6a6bc6ee5548d366a3f4c6c7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/04255de21db75e90b170040f4d1b457ba721e7a5", - "reference": "04255de21db75e90b170040f4d1b457ba721e7a5", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/77a863a920122e2c6a6bc6ee5548d366a3f4c6c7", + "reference": "77a863a920122e2c6a6bc6ee5548d366a3f4c6c7", "shasum": "" }, "require": { @@ -4701,6 +4701,7 @@ }, "require-dev": { "laravel/pint": "1.*", + "phpstan/phpstan": "2.*", "phpunit/phpunit": "9.*" }, "type": "library", @@ -4723,9 +4724,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/0.7.12" + "source": "https://github.com/utopia-php/platform/tree/0.7.13" }, - "time": "2025-09-05T15:53:12+00:00" + "time": "2025-12-08T10:02:40+00:00" }, { "name": "utopia-php/pools", @@ -7930,16 +7931,16 @@ }, { "name": "symfony/console", - "version": "v8.0.0", + "version": "v8.0.1", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "307d3cf852f5ead3618ac60ecbedbdd512c348b1" + "reference": "fcb73f69d655b48fcb894a262f074218df08bd58" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/307d3cf852f5ead3618ac60ecbedbdd512c348b1", - "reference": "307d3cf852f5ead3618ac60ecbedbdd512c348b1", + "url": "https://api.github.com/repos/symfony/console/zipball/fcb73f69d655b48fcb894a262f074218df08bd58", + "reference": "fcb73f69d655b48fcb894a262f074218df08bd58", "shasum": "" }, "require": { @@ -7996,7 +7997,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v8.0.0" + "source": "https://github.com/symfony/console/tree/v8.0.1" }, "funding": [ { @@ -8016,20 +8017,20 @@ "type": "tidelift" } ], - "time": "2025-11-21T13:19:49+00:00" + "time": "2025-12-05T15:25:33+00:00" }, { "name": "symfony/filesystem", - "version": "v8.0.0", + "version": "v8.0.1", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "7fc96ae83372620eaba3826874f46e26295768ca" + "reference": "d937d400b980523dc9ee946bb69972b5e619058d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/7fc96ae83372620eaba3826874f46e26295768ca", - "reference": "7fc96ae83372620eaba3826874f46e26295768ca", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/d937d400b980523dc9ee946bb69972b5e619058d", + "reference": "d937d400b980523dc9ee946bb69972b5e619058d", "shasum": "" }, "require": { @@ -8066,7 +8067,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v8.0.0" + "source": "https://github.com/symfony/filesystem/tree/v8.0.1" }, "funding": [ { @@ -8086,7 +8087,7 @@ "type": "tidelift" } ], - "time": "2025-11-05T14:36:47+00:00" + "time": "2025-12-01T09:13:36+00:00" }, { "name": "symfony/finder", @@ -8624,16 +8625,16 @@ }, { "name": "symfony/string", - "version": "v8.0.0", + "version": "v8.0.1", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "f929eccf09531078c243df72398560e32fa4cf4f" + "reference": "ba65a969ac918ce0cc3edfac6cdde847eba231dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/f929eccf09531078c243df72398560e32fa4cf4f", - "reference": "f929eccf09531078c243df72398560e32fa4cf4f", + "url": "https://api.github.com/repos/symfony/string/zipball/ba65a969ac918ce0cc3edfac6cdde847eba231dc", + "reference": "ba65a969ac918ce0cc3edfac6cdde847eba231dc", "shasum": "" }, "require": { @@ -8690,7 +8691,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v8.0.0" + "source": "https://github.com/symfony/string/tree/v8.0.1" }, "funding": [ { @@ -8710,7 +8711,7 @@ "type": "tidelift" } ], - "time": "2025-09-11T14:37:55+00:00" + "time": "2025-12-01T09:13:36+00:00" }, { "name": "textalk/websocket", diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 74c91c193d..0ad8c5ac9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -241,17 +241,17 @@ abstract class ScheduleBase extends Action } foreach ($this->schedules as $sequence => $schedule) { - $project = $map[$schedule['projectId']]; + $project = $map[$schedule['projectId']] ?? null; - // In case the resource is blocked. - if ($isResourceBlocked($project, $collectionId, $schedule['resourceId'])) { - Console::error("Resource blocked: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); + if ($project === null || $project->isEmpty()) { + Console::error("Project not found: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); unset($this->schedules[$sequence]); continue; } - if (empty($project)) { - Console::error("Project not found: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); + // In case the resource is blocked. + if ($isResourceBlocked($project, $collectionId, $schedule['resourceId'])) { + Console::error("Resource blocked: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); unset($this->schedules[$sequence]); continue; } @@ -268,7 +268,7 @@ abstract class ScheduleBase extends Action continue; } - if (empty($resource)) { + if ($resource->isEmpty()) { Console::error("Resource not found: projectId::{$schedule['projectId']} resourceId::{$schedule['resourceId']}"); unset($this->schedules[$sequence]); continue; From 7cdb7af07ce34d293fc54eeda6011936da279076 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 16 Dec 2025 20:06:38 +1300 Subject: [PATCH 6/6] Remove early return to ensure cycle timestamp is updated --- src/Appwrite/Platform/Tasks/ScheduleBase.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 0ad8c5ac9a..3853f0d9b3 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -195,7 +195,6 @@ abstract class ScheduleBase extends Action } if (empty($this->schedules)) { Console::success("No resources found"); - return; } // On initial load: load all projects from all schedules