From 95f49559d08eb50b06267f5cd3ee805336c50ed4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 12:43:37 +0300 Subject: [PATCH 01/10] add finally --- src/Appwrite/Platform/Workers/StatsResources.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index da8c086bf4..655ebfbd52 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -206,6 +206,8 @@ class StatsResources extends Action $this->writeDocuments($dbForLogs, $project); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); + } finally { + $this->documents = []; } Console::info('End of count for: ' . $project->getId()); From de9e177bc84cb5ddc1a31fb10ede3d5260161e6d Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 17:23:33 +0300 Subject: [PATCH 02/10] sort and hardcode bachSize --- .../Platform/Workers/StatsResources.php | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 655ebfbd52..d26f4c06f3 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -206,8 +206,6 @@ class StatsResources extends Action $this->writeDocuments($dbForLogs, $project); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); - } finally { - $this->documents = []; } Console::info('End of count for: ' . $project->getId()); @@ -436,18 +434,32 @@ class StatsResources extends Action { $message = 'Stats writeDocuments project: ' . $project->getId() . '(' . $project->getSequence() . ')'; + // sort by unique index key to make + usort($this->documents, function($a, $b) { + // metric DESC + $cmp = strcmp($b['metric'], $a['metric']); + if ($cmp !== 0) return $cmp; + + // period ASC + $cmp = strcmp($a['period'], $b['period']); + if ($cmp !== 0) return $cmp; + + // time ASC + if ($a['time'] === $b['time']) return 0; + return ($a['time'] < $b['time']) ? -1 : 1; + }); + try { $dbForLogs->createOrUpdateDocuments( 'stats', - $this->documents + $this->documents, + 10 // See if this make an effect ); Console::success($message . ' | Documents: ' . count($this->documents)); } catch (\Throwable $e) { Console::error('Error: ' . $message . ' | Exception: ' . $e->getMessage()); throw $e; - } finally { - $this->documents = []; } } } From 494fe274b08f3ee3e5c5856aa7598f251149c891 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 17:43:37 +0300 Subject: [PATCH 03/10] sort as strings --- src/Appwrite/Platform/Workers/StatsResources.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index d26f4c06f3..932b5072f9 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -444,9 +444,8 @@ class StatsResources extends Action $cmp = strcmp($a['period'], $b['period']); if ($cmp !== 0) return $cmp; - // time ASC - if ($a['time'] === $b['time']) return 0; - return ($a['time'] < $b['time']) ? -1 : 1; + // time ASC (string comparison is fine) + return strcmp($a['time'], $b['time']); }); try { From b44051e0a2377a8a1647096adf3213cfea65e30f Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 17:48:58 +0300 Subject: [PATCH 04/10] Respect time nulls --- src/Appwrite/Platform/Workers/StatsResources.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 932b5072f9..f7ee999a94 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -434,7 +434,7 @@ class StatsResources extends Action { $message = 'Stats writeDocuments project: ' . $project->getId() . '(' . $project->getSequence() . ')'; - // sort by unique index key to make + // sort by unique index key reduce locks usort($this->documents, function($a, $b) { // metric DESC $cmp = strcmp($b['metric'], $a['metric']); @@ -444,10 +444,15 @@ class StatsResources extends Action $cmp = strcmp($a['period'], $b['period']); if ($cmp !== 0) return $cmp; - // time ASC (string comparison is fine) + // time ASC, NULLs first + if ($a['time'] === null && $b['time'] === null) return 0; + if ($a['time'] === null) return -1; + if ($b['time'] === null) return 1; + return strcmp($a['time'], $b['time']); }); +var_dump($this->documents); try { $dbForLogs->createOrUpdateDocuments( 'stats', From 6ed4590e0f2b522188fb85cf1b5e0e9c2e0fd248 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 18:04:35 +0300 Subject: [PATCH 05/10] Respect time nulls --- .../Platform/Workers/StatsResources.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index f7ee999a94..daecabf16a 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -434,25 +434,26 @@ class StatsResources extends Action { $message = 'Stats writeDocuments project: ' . $project->getId() . '(' . $project->getSequence() . ')'; - // sort by unique index key reduce locks - usort($this->documents, function($a, $b) { - // metric DESC + /** + * sort by unique index key reduce locks/deadlocks + */ + usort($this->documents, function ($a, $b) { + // Metric DESC $cmp = strcmp($b['metric'], $a['metric']); if ($cmp !== 0) return $cmp; - // period ASC + // Period ASC $cmp = strcmp($a['period'], $b['period']); if ($cmp !== 0) return $cmp; - // time ASC, NULLs first - if ($a['time'] === null && $b['time'] === null) return 0; - if ($a['time'] === null) return -1; + // Time ASC, NULLs first + if ($a['time'] === null) return ($b['time'] === null) ? 0 : -1; if ($b['time'] === null) return 1; return strcmp($a['time'], $b['time']); }); -var_dump($this->documents); + var_dump($this->documents); try { $dbForLogs->createOrUpdateDocuments( 'stats', From a31190422a7b3be745d1114e8c86bba071dd7636 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 09:41:17 +0300 Subject: [PATCH 06/10] formatting --- .../Platform/Workers/StatsResources.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index daecabf16a..0c8f11c07b 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -440,25 +440,31 @@ class StatsResources extends Action usort($this->documents, function ($a, $b) { // Metric DESC $cmp = strcmp($b['metric'], $a['metric']); - if ($cmp !== 0) return $cmp; + if ($cmp !== 0) { + return $cmp; + } // Period ASC $cmp = strcmp($a['period'], $b['period']); - if ($cmp !== 0) return $cmp; + if ($cmp !== 0) { + return $cmp; + } // Time ASC, NULLs first - if ($a['time'] === null) return ($b['time'] === null) ? 0 : -1; - if ($b['time'] === null) return 1; + if ($a['time'] === null) { + return ($b['time'] === null) ? 0 : -1; + } + if ($b['time'] === null) { + return 1; + } return strcmp($a['time'], $b['time']); }); - var_dump($this->documents); try { $dbForLogs->createOrUpdateDocuments( 'stats', $this->documents, - 10 // See if this make an effect ); Console::success($message . ' | Documents: ' . count($this->documents)); From 5740eb89012d474142c25646eed696ba3f55441a Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 10:04:42 +0300 Subject: [PATCH 07/10] Add sorting --- src/Appwrite/Platform/Workers/StatsUsage.php | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 3610381d5a..991fed633a 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -424,6 +424,34 @@ class StatsUsage extends Action try { $dbForProject = $getProjectDB($projectStats['project']); Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); + + /** + * Sort by unique index key reduce locks/deadlocks + */ + usort($projectStats['stats'], function ($a, $b) { + // Metric DESC + $cmp = strcmp($b['metric'], $a['metric']); + if ($cmp !== 0) { + return $cmp; + } + + // Period ASC + $cmp = strcmp($a['period'], $b['period']); + if ($cmp !== 0) { + return $cmp; + } + + // Time ASC, NULLs first + if ($a['time'] === null) { + return ($b['time'] === null) ? 0 : -1; + } + if ($b['time'] === null) { + return 1; + } + + return strcmp($a['time'], $b['time']); + }); + $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); Console::success('Batch successfully written to DB'); @@ -468,6 +496,42 @@ class StatsUsage extends Action try { Console::log('Processing batch with ' . count($this->statDocuments) . ' stats'); + + /** + * Sort by UNIQUE KEY "_key_metric_period_time" ("_tenant","metric" DESC,"period","time") + * Here we sort by _tenant as well because of setTenantPerDocument + */ + + usort($this->statDocuments, function ($a, $b) { + // Tenant ASC + $cmp = $a['_tenant'] <=> $b['_tenant']; + if ($cmp !== 0) { + return $cmp; + } + + // Metric DESC + $cmp = strcmp($b['metric'], $a['metric']); + if ($cmp !== 0) { + return $cmp; + } + + // Period ASC + $cmp = strcmp($a['period'], $b['period']); + if ($cmp !== 0) { + return $cmp; + } + + // Time ASC, NULLs first + if ($a['time'] === null) { + return ($b['time'] === null) ? 0 : -1; + } + if ($b['time'] === null) { + return 1; + } + + return strcmp($a['time'], $b['time']); + }); + $dbForLogs->createOrUpdateDocumentsWithIncrease( 'stats', 'value', From 84aaaf810f7716e1cf983c0b87f25eaf0aa7cadd Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 10:07:45 +0300 Subject: [PATCH 08/10] Question --- src/Appwrite/Platform/Workers/StatsUsage.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 991fed633a..ba72f7f30c 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -454,10 +454,10 @@ class StatsUsage extends Action $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); Console::success('Batch successfully written to DB'); - - unset($this->projects[$sequence]); } catch (Throwable $e) { Console::error('Error processing stats: ' . $e->getMessage()); + } finally { + unset($this->projects[$sequence]); } } @@ -538,6 +538,11 @@ class StatsUsage extends Action $this->statDocuments ); Console::success('Usage logs pushed to Logs DB'); + + /** + * todo: Do we need to unset $this->statDocuments? + */ + } catch (Throwable $th) { Console::error($th->getMessage()); } From e673d405d685b52e544817aceca57b6c7e8779b4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 10:14:16 +0300 Subject: [PATCH 09/10] formatting? --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index ba72f7f30c..9d998d05bc 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -538,7 +538,7 @@ class StatsUsage extends Action $this->statDocuments ); Console::success('Usage logs pushed to Logs DB'); - + /** * todo: Do we need to unset $this->statDocuments? */ From 7d1a1d3ef479160b05fb24df665ad3ade5e7d35d Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 17 Sep 2025 08:45:23 +0300 Subject: [PATCH 10/10] use $tenant --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 9d998d05bc..bd34d53b00 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -504,7 +504,7 @@ class StatsUsage extends Action usort($this->statDocuments, function ($a, $b) { // Tenant ASC - $cmp = $a['_tenant'] <=> $b['_tenant']; + $cmp = $a['$tenant'] <=> $b['$tenant']; if ($cmp !== 0) { return $cmp; }