mirror of
https://github.com/appwrite/appwrite
synced 2026-05-23 08:58:35 +00:00
Update composer.lock and refactor ScheduleBase class for improved schedule processing
This commit is contained in:
shimon
parent
379b85bb7b
commit
d483fbd43b
2 changed files with 84 additions and 62 deletions
28
composer.lock
generated
28
composer.lock
generated
|
|
@ -4952,16 +4952,16 @@
|
|||
},
|
||||
{
|
||||
"name": "utopia-php/storage",
|
||||
"version": "0.18.14",
|
||||
"version": "0.18.16",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/storage.git",
|
||||
"reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00"
|
||||
"reference": "0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/storage/zipball/4f14ec952c6f4006dd0613e55bbf7631814fbc00",
|
||||
"reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00",
|
||||
"url": "https://api.github.com/repos/utopia-php/storage/zipball/0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f",
|
||||
"reference": "0c7b8ad68de8e1eb23ccc8af9f27a30eb832930f",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
|
@ -5004,9 +5004,9 @@
|
|||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/storage/issues",
|
||||
"source": "https://github.com/utopia-php/storage/tree/0.18.14"
|
||||
"source": "https://github.com/utopia-php/storage/tree/0.18.16"
|
||||
},
|
||||
"time": "2025-10-07T10:21:47+00:00"
|
||||
"time": "2025-12-03T02:15:45+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/swoole",
|
||||
|
|
@ -6663,16 +6663,16 @@
|
|||
},
|
||||
{
|
||||
"name": "phpunit/phpunit",
|
||||
"version": "9.6.29",
|
||||
"version": "9.6.30",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/sebastianbergmann/phpunit.git",
|
||||
"reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3"
|
||||
"reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/9ecfec57835a5581bc888ea7e13b51eb55ab9dd3",
|
||||
"reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3",
|
||||
"url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/b69489b312503bf8fa6d75a76916919d7d2fa6d4",
|
||||
"reference": "b69489b312503bf8fa6d75a76916919d7d2fa6d4",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
|
@ -6746,7 +6746,7 @@
|
|||
"support": {
|
||||
"issues": "https://github.com/sebastianbergmann/phpunit/issues",
|
||||
"security": "https://github.com/sebastianbergmann/phpunit/security/policy",
|
||||
"source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.29"
|
||||
"source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.30"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
|
|
@ -6770,7 +6770,7 @@
|
|||
"type": "tidelift"
|
||||
}
|
||||
],
|
||||
"time": "2025-09-24T06:29:11+00:00"
|
||||
"time": "2025-12-01T07:35:08+00:00"
|
||||
},
|
||||
{
|
||||
"name": "psr/cache",
|
||||
|
|
@ -8942,7 +8942,7 @@
|
|||
],
|
||||
"aliases": [],
|
||||
"minimum-stability": "stable",
|
||||
"stability-flags": {},
|
||||
"stability-flags": [],
|
||||
"prefer-stable": false,
|
||||
"prefer-lowest": false,
|
||||
"platform": {
|
||||
|
|
@ -8966,5 +8966,5 @@
|
|||
"platform-overrides": {
|
||||
"php": "8.3"
|
||||
},
|
||||
"plugin-api-version": "2.6.0"
|
||||
"plugin-api-version": "2.2.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,36 +115,7 @@ abstract class ScheduleBase extends Action
|
|||
|
||||
private function collectSchedules(Database $dbForPlatform, callable $getProjectDB, string &$lastSyncUpdate, callable $isResourceBlocked): void
|
||||
{
|
||||
// If we haven't synced yet, load all active schedules
|
||||
$initialLoad = $lastSyncUpdate === "0";
|
||||
|
||||
/**
|
||||
* Extract only necessary attributes to lower memory used.
|
||||
*
|
||||
* @return array
|
||||
* @throws Exception
|
||||
* @var Document $schedule
|
||||
*/
|
||||
$getSchedule = function (Document $schedule) use ($dbForPlatform, $getProjectDB): array {
|
||||
$project = $dbForPlatform->getDocument('projects', $schedule->getAttribute('projectId'));
|
||||
|
||||
$resource = $getProjectDB($project)->getDocument(
|
||||
static::getCollectionId(),
|
||||
$schedule->getAttribute('resourceId')
|
||||
);
|
||||
|
||||
return [
|
||||
'$sequence' => $schedule->getSequence(),
|
||||
'$id' => $schedule->getId(),
|
||||
'resourceId' => $schedule->getAttribute('resourceId'),
|
||||
'schedule' => $schedule->getAttribute('schedule'),
|
||||
'active' => $schedule->getAttribute('active'),
|
||||
'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'),
|
||||
'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
|
||||
'resource' => $resource, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
|
||||
];
|
||||
};
|
||||
|
||||
$loadStart = microtime(true);
|
||||
$time = DateTime::now();
|
||||
|
||||
|
|
@ -152,6 +123,9 @@ abstract class ScheduleBase extends Action
|
|||
$sum = $limit;
|
||||
$total = 0;
|
||||
$latestDocument = null;
|
||||
$collectionId = static::getCollectionId();
|
||||
|
||||
$schedulesToProcess = [];
|
||||
|
||||
while ($sum === $limit) {
|
||||
$paginationQueries = [Query::limit($limit)];
|
||||
|
|
@ -160,8 +134,6 @@ abstract class ScheduleBase extends Action
|
|||
$paginationQueries[] = Query::cursorAfter($latestDocument);
|
||||
}
|
||||
|
||||
// Temporarly accepting both 'fra' and 'default'
|
||||
// When all migrated, only use _APP_REGION with 'default' as default value
|
||||
$regions = [System::getEnv('_APP_REGION', 'default')];
|
||||
if (!in_array('default', $regions)) {
|
||||
$regions[] = 'default';
|
||||
|
|
@ -179,7 +151,6 @@ abstract class ScheduleBase extends Action
|
|||
$paginationQueries[] = Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate);
|
||||
}
|
||||
|
||||
$collectionId = static::getCollectionId();
|
||||
$schedules = $dbForPlatform->find('schedules', $paginationQueries);
|
||||
$sum = count($schedules);
|
||||
$total += $sum;
|
||||
|
|
@ -189,32 +160,83 @@ abstract class ScheduleBase extends Action
|
|||
$updated = strtotime($existing['resourceUpdatedAt'] ?? '0') !== strtotime($schedule['resourceUpdatedAt'] ?? '0');
|
||||
|
||||
if ($existing === null || $updated) {
|
||||
try {
|
||||
$candidate = $getSchedule($schedule);
|
||||
} catch (\Throwable $th) {
|
||||
Console::error("Failed to load schedule for project {$schedule['projectId']} {$collectionId} {$schedule['resourceId']}");
|
||||
Console::error($th->getMessage());
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!$candidate['active']) {
|
||||
// Early filtering: skip if not active (only for updates, initial load already filters)
|
||||
if (!$initialLoad && !$schedule->getAttribute('active', true)) {
|
||||
unset($this->schedules[$schedule->getSequence()]);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($isResourceBlocked($candidate['project'], $collectionId, $candidate['resourceId'])) {
|
||||
unset($this->schedules[$schedule->getSequence()]);
|
||||
continue;
|
||||
}
|
||||
|
||||
Console::info("Updating: {$schedule['resourceType']}::{$schedule['resourceId']}");
|
||||
$this->schedules[$schedule->getSequence()] = $candidate;
|
||||
$schedulesToProcess[] = $schedule;
|
||||
}
|
||||
}
|
||||
|
||||
$latestDocument = \end($schedules);
|
||||
}
|
||||
|
||||
if (empty($schedulesToProcess)) {
|
||||
$lastSyncUpdate = $time;
|
||||
$duration = microtime(true) - $loadStart;
|
||||
$this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]);
|
||||
$this->collectSchedulesTelemetryCount->record($total, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]);
|
||||
Console::success("{$total} resources were loaded in " . $duration . " seconds");
|
||||
return;
|
||||
}
|
||||
|
||||
// Cache projects to avoid fetching the same project multiple times
|
||||
$projectsCache = [];
|
||||
|
||||
foreach ($schedulesToProcess as $schedule) {
|
||||
$projectId = $schedule->getAttribute('projectId');
|
||||
|
||||
// Fetch project if not already cached
|
||||
if (!isset($projectsCache[$projectId])) {
|
||||
try {
|
||||
$projectsCache[$projectId] = $dbForPlatform->getDocument('projects', $projectId);
|
||||
} catch (\Throwable $th) {
|
||||
Console::error("Failed to load project {$projectId}: " . $th->getMessage());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$project = $projectsCache[$projectId];
|
||||
if ($project === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$resourceId = $schedule->getAttribute('resourceId');
|
||||
$resource = $dbForProject->getDocument($collectionId, $resourceId);
|
||||
|
||||
$candidate = [
|
||||
'$sequence' => $schedule->getSequence(),
|
||||
'$id' => $schedule->getId(),
|
||||
'resourceId' => $resourceId,
|
||||
'schedule' => $schedule->getAttribute('schedule'),
|
||||
'active' => $schedule->getAttribute('active'),
|
||||
'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'),
|
||||
'project' => $project,
|
||||
'resource' => $resource,
|
||||
];
|
||||
|
||||
// Early filtering checks
|
||||
if (!$candidate['active']) {
|
||||
unset($this->schedules[$schedule->getSequence()]);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($isResourceBlocked($candidate['project'], $collectionId, $candidate['resourceId'])) {
|
||||
unset($this->schedules[$schedule->getSequence()]);
|
||||
continue;
|
||||
}
|
||||
|
||||
Console::info("loading: project:: " . $candidate['project']->getId() . " " . static::getSupportedResource() . "::{$candidate['resourceId']}");
|
||||
$this->schedules[$schedule->getSequence()] = $candidate;
|
||||
} catch (\Throwable $th) {
|
||||
Console::error("Failed to load schedule for project {$project->getId()} {$collectionId} {$schedule->getAttribute('resourceId')}: " . $th->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
$lastSyncUpdate = $time;
|
||||
$duration = microtime(true) - $loadStart;
|
||||
$this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]);
|
||||
|
|
|
|||
Loading…
Reference in a new issue