From 95f49559d08eb50b06267f5cd3ee805336c50ed4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 12:43:37 +0300 Subject: [PATCH 01/26] add finally --- src/Appwrite/Platform/Workers/StatsResources.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index da8c086bf4..655ebfbd52 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -206,6 +206,8 @@ class StatsResources extends Action $this->writeDocuments($dbForLogs, $project); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); + } finally { + $this->documents = []; } Console::info('End of count for: ' . $project->getId()); From de9e177bc84cb5ddc1a31fb10ede3d5260161e6d Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 17:23:33 +0300 Subject: [PATCH 02/26] sort and hardcode bachSize --- .../Platform/Workers/StatsResources.php | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 655ebfbd52..d26f4c06f3 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -206,8 +206,6 @@ class StatsResources extends Action $this->writeDocuments($dbForLogs, $project); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); - } finally { - $this->documents = []; } Console::info('End of count for: ' . $project->getId()); @@ -436,18 +434,32 @@ class StatsResources extends Action { $message = 'Stats writeDocuments project: ' . $project->getId() . '(' . $project->getSequence() . ')'; + // sort by unique index key to make + usort($this->documents, function($a, $b) { + // metric DESC + $cmp = strcmp($b['metric'], $a['metric']); + if ($cmp !== 0) return $cmp; + + // period ASC + $cmp = strcmp($a['period'], $b['period']); + if ($cmp !== 0) return $cmp; + + // time ASC + if ($a['time'] === $b['time']) return 0; + return ($a['time'] < $b['time']) ? -1 : 1; + }); + try { $dbForLogs->createOrUpdateDocuments( 'stats', - $this->documents + $this->documents, + 10 // See if this make an effect ); Console::success($message . ' | Documents: ' . count($this->documents)); } catch (\Throwable $e) { Console::error('Error: ' . $message . ' | Exception: ' . $e->getMessage()); throw $e; - } finally { - $this->documents = []; } } } From 494fe274b08f3ee3e5c5856aa7598f251149c891 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 17:43:37 +0300 Subject: [PATCH 03/26] sort as strings --- src/Appwrite/Platform/Workers/StatsResources.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index d26f4c06f3..932b5072f9 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -444,9 +444,8 @@ class StatsResources extends Action $cmp = strcmp($a['period'], $b['period']); if ($cmp !== 0) return $cmp; - // time ASC - if ($a['time'] === $b['time']) return 0; - return ($a['time'] < $b['time']) ? -1 : 1; + // time ASC (string comparison is fine) + return strcmp($a['time'], $b['time']); }); try { From b44051e0a2377a8a1647096adf3213cfea65e30f Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 17:48:58 +0300 Subject: [PATCH 04/26] Respect time nulls --- src/Appwrite/Platform/Workers/StatsResources.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 932b5072f9..f7ee999a94 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -434,7 +434,7 @@ class StatsResources extends Action { $message = 'Stats writeDocuments project: ' . $project->getId() . '(' . $project->getSequence() . ')'; - // sort by unique index key to make + // sort by unique index key reduce locks usort($this->documents, function($a, $b) { // metric DESC $cmp = strcmp($b['metric'], $a['metric']); @@ -444,10 +444,15 @@ class StatsResources extends Action $cmp = strcmp($a['period'], $b['period']); if ($cmp !== 0) return $cmp; - // time ASC (string comparison is fine) + // time ASC, NULLs first + if ($a['time'] === null && $b['time'] === null) return 0; + if ($a['time'] === null) return -1; + if ($b['time'] === null) return 1; + return strcmp($a['time'], $b['time']); }); +var_dump($this->documents); try { $dbForLogs->createOrUpdateDocuments( 'stats', From 6ed4590e0f2b522188fb85cf1b5e0e9c2e0fd248 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 18:04:35 +0300 Subject: [PATCH 05/26] Respect time nulls --- .../Platform/Workers/StatsResources.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index f7ee999a94..daecabf16a 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -434,25 +434,26 @@ class StatsResources extends Action { $message = 'Stats writeDocuments project: ' . $project->getId() . '(' . $project->getSequence() . ')'; - // sort by unique index key reduce locks - usort($this->documents, function($a, $b) { - // metric DESC + /** + * sort by unique index key reduce locks/deadlocks + */ + usort($this->documents, function ($a, $b) { + // Metric DESC $cmp = strcmp($b['metric'], $a['metric']); if ($cmp !== 0) return $cmp; - // period ASC + // Period ASC $cmp = strcmp($a['period'], $b['period']); if ($cmp !== 0) return $cmp; - // time ASC, NULLs first - if ($a['time'] === null && $b['time'] === null) return 0; - if ($a['time'] === null) return -1; + // Time ASC, NULLs first + if ($a['time'] === null) return ($b['time'] === null) ? 0 : -1; if ($b['time'] === null) return 1; return strcmp($a['time'], $b['time']); }); -var_dump($this->documents); + var_dump($this->documents); try { $dbForLogs->createOrUpdateDocuments( 'stats', From a31190422a7b3be745d1114e8c86bba071dd7636 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 09:41:17 +0300 Subject: [PATCH 06/26] formatting --- .../Platform/Workers/StatsResources.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index daecabf16a..0c8f11c07b 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -440,25 +440,31 @@ class StatsResources extends Action usort($this->documents, function ($a, $b) { // Metric DESC $cmp = strcmp($b['metric'], $a['metric']); - if ($cmp !== 0) return $cmp; + if ($cmp !== 0) { + return $cmp; + } // Period ASC $cmp = strcmp($a['period'], $b['period']); - if ($cmp !== 0) return $cmp; + if ($cmp !== 0) { + return $cmp; + } // Time ASC, NULLs first - if ($a['time'] === null) return ($b['time'] === null) ? 0 : -1; - if ($b['time'] === null) return 1; + if ($a['time'] === null) { + return ($b['time'] === null) ? 0 : -1; + } + if ($b['time'] === null) { + return 1; + } return strcmp($a['time'], $b['time']); }); - var_dump($this->documents); try { $dbForLogs->createOrUpdateDocuments( 'stats', $this->documents, - 10 // See if this make an effect ); Console::success($message . ' | Documents: ' . count($this->documents)); From 5740eb89012d474142c25646eed696ba3f55441a Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 10:04:42 +0300 Subject: [PATCH 07/26] Add sorting --- src/Appwrite/Platform/Workers/StatsUsage.php | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 3610381d5a..991fed633a 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -424,6 +424,34 @@ class StatsUsage extends Action try { $dbForProject = $getProjectDB($projectStats['project']); Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); + + /** + * Sort by unique index key reduce locks/deadlocks + */ + usort($projectStats['stats'], function ($a, $b) { + // Metric DESC + $cmp = strcmp($b['metric'], $a['metric']); + if ($cmp !== 0) { + return $cmp; + } + + // Period ASC + $cmp = strcmp($a['period'], $b['period']); + if ($cmp !== 0) { + return $cmp; + } + + // Time ASC, NULLs first + if ($a['time'] === null) { + return ($b['time'] === null) ? 0 : -1; + } + if ($b['time'] === null) { + return 1; + } + + return strcmp($a['time'], $b['time']); + }); + $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); Console::success('Batch successfully written to DB'); @@ -468,6 +496,42 @@ class StatsUsage extends Action try { Console::log('Processing batch with ' . count($this->statDocuments) . ' stats'); + + /** + * Sort by UNIQUE KEY "_key_metric_period_time" ("_tenant","metric" DESC,"period","time") + * Here we sort by _tenant as well because of setTenantPerDocument + */ + + usort($this->statDocuments, function ($a, $b) { + // Tenant ASC + $cmp = $a['_tenant'] <=> $b['_tenant']; + if ($cmp !== 0) { + return $cmp; + } + + // Metric DESC + $cmp = strcmp($b['metric'], $a['metric']); + if ($cmp !== 0) { + return $cmp; + } + + // Period ASC + $cmp = strcmp($a['period'], $b['period']); + if ($cmp !== 0) { + return $cmp; + } + + // Time ASC, NULLs first + if ($a['time'] === null) { + return ($b['time'] === null) ? 0 : -1; + } + if ($b['time'] === null) { + return 1; + } + + return strcmp($a['time'], $b['time']); + }); + $dbForLogs->createOrUpdateDocumentsWithIncrease( 'stats', 'value', From 84aaaf810f7716e1cf983c0b87f25eaf0aa7cadd Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 10:07:45 +0300 Subject: [PATCH 08/26] Question --- src/Appwrite/Platform/Workers/StatsUsage.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 991fed633a..ba72f7f30c 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -454,10 +454,10 @@ class StatsUsage extends Action $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); Console::success('Batch successfully written to DB'); - - unset($this->projects[$sequence]); } catch (Throwable $e) { Console::error('Error processing stats: ' . $e->getMessage()); + } finally { + unset($this->projects[$sequence]); } } @@ -538,6 +538,11 @@ class StatsUsage extends Action $this->statDocuments ); Console::success('Usage logs pushed to Logs DB'); + + /** + * todo: Do we need to unset $this->statDocuments? + */ + } catch (Throwable $th) { Console::error($th->getMessage()); } From e673d405d685b52e544817aceca57b6c7e8779b4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 11 Sep 2025 10:14:16 +0300 Subject: [PATCH 09/26] formatting? --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index ba72f7f30c..9d998d05bc 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -538,7 +538,7 @@ class StatsUsage extends Action $this->statDocuments ); Console::success('Usage logs pushed to Logs DB'); - + /** * todo: Do we need to unset $this->statDocuments? */ From ef3d38c5a65fb0bbbfd209238587092c43b3023d Mon Sep 17 00:00:00 2001 From: Divyansha Dubey Date: Fri, 12 Sep 2025 17:48:15 +0400 Subject: [PATCH 10/26] fixed db stats env var --- .../Platform/Modules/Databases/Http/Databases/Usage/Get.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php index 4770d727a0..c9de9d5217 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php @@ -77,8 +77,8 @@ class Get extends Action str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_COLLECTIONS), str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_DOCUMENTS), str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_STORAGE), - str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASES_OPERATIONS_READS), - str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASES_OPERATIONS_WRITES) + str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), + str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES) ]; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { From 66503a4e806048b540d334474558ba9a121a5a49 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:15:56 +0100 Subject: [PATCH 11/26] fix: api worker-stop --- app/http.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/http.php b/app/http.php index 30f4013821..1bd3e97e69 100644 --- a/app/http.php +++ b/app/http.php @@ -10,6 +10,7 @@ use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; use Swoole\Process; use Swoole\Table; +use Swoole\Timer; use Utopia\App; use Utopia\Audit\Audit; use Utopia\CLI\Console; @@ -156,11 +157,16 @@ $http->on(Constant::EVENT_WORKER_START, function ($server, $workerId) { Console::success('Worker ' . ++$workerId . ' started successfully'); }); -$http->on(Constant::EVENT_BEFORE_RELOAD, function ($server, $workerId) { +$http->on(Constant::EVENT_WORKER_STOP, function ($server, $workerId) { + Timer::clearAll(); + Console::success('Worker ' . ++$workerId . ' stopped successfully'); +}); + +$http->on(Constant::EVENT_BEFORE_RELOAD, function ($server) { Console::success('Starting reload...'); }); -$http->on(Constant::EVENT_AFTER_RELOAD, function ($server, $workerId) { +$http->on(Constant::EVENT_AFTER_RELOAD, function ($server) { Console::success('Reload completed...'); }); @@ -550,7 +556,7 @@ $http->on(Constant::EVENT_TASK, function () use ($register, $domains) { /** @var Utopia\Database\Database $dbForPlatform */ $dbForPlatform = $app->getResource('dbForPlatform'); - Console::loop(function () use ($dbForPlatform, $domains, &$lastSyncUpdate) { + Timer::tick(DOMAIN_SYNC_TIMER * 1000, function () use ($dbForPlatform, $domains, &$lastSyncUpdate) { try { $time = DateTime::now(); $limit = 1000; @@ -589,8 +595,6 @@ $http->on(Constant::EVENT_TASK, function () use ($register, $domains) { } catch (Throwable $th) { Console::error($th->getMessage()); } - }, DOMAIN_SYNC_TIMER, 0, function ($error) { - Console::error($error); }); }); From 081e2aeb6e1ce80006f1cd24b4cf737f1e9a8367 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 16 Sep 2025 17:51:22 +1200 Subject: [PATCH 12/26] Throw appropriate 400s from request filters --- src/Appwrite/Utopia/Request/Filters/V16.php | 2 - src/Appwrite/Utopia/Request/Filters/V17.php | 54 +++++++++++++-------- src/Appwrite/Utopia/Request/Filters/V20.php | 35 ++++++++----- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/src/Appwrite/Utopia/Request/Filters/V16.php b/src/Appwrite/Utopia/Request/Filters/V16.php index 51b05359b3..55db1f4756 100644 --- a/src/Appwrite/Utopia/Request/Filters/V16.php +++ b/src/Appwrite/Utopia/Request/Filters/V16.php @@ -11,8 +11,6 @@ class V16 extends Filter { switch ($model) { case 'functions.create': - $content['commands'] = $this->getCommands($content['runtime'] ?? ''); - break; case 'functions.update': $content['commands'] = $this->getCommands($content['runtime'] ?? ''); break; diff --git a/src/Appwrite/Utopia/Request/Filters/V17.php b/src/Appwrite/Utopia/Request/Filters/V17.php index 83ec62a168..2cdf3973b2 100644 --- a/src/Appwrite/Utopia/Request/Filters/V17.php +++ b/src/Appwrite/Utopia/Request/Filters/V17.php @@ -2,6 +2,7 @@ namespace Appwrite\Utopia\Request\Filters; +use Appwrite\Extend\Exception; use Appwrite\Utopia\Request\Filter; use Utopia\Database\Query; @@ -67,9 +68,9 @@ class V17 extends Filter foreach ($content['queries'] as $query) { try { $query = $this->parseQuery($query); - $parsed[] = json_encode(array_filter($query->toArray())); + $parsed[] = \json_encode(\array_filter($query->toArray())); } catch (\Throwable $th) { - throw new \Exception("Invalid query: {$query}", previous: $th); + throw new Exception(Exception::GENERAL_QUERY_INVALID, $th->getMessage()); } } @@ -83,6 +84,7 @@ class V17 extends Filter { // Init empty vars we fill later $method = ''; + $attribute = null; $params = []; // Separate method from filter @@ -92,7 +94,7 @@ class V17 extends Filter throw new \Exception('Invalid query'); } - $method = mb_substr($filter, 0, $paramsStart); + $method = \mb_substr($filter, 0, $paramsStart); // Separate params from filter $paramsEnd = \strlen($filter) - 1; // -1 to ignore ) @@ -103,14 +105,13 @@ class V17 extends Filter throw new \Exception('Invalid query method'); } - $currentParam = ""; // We build param here before pushing when it's ended + $currentParam = ''; // We build param here before pushing when it's ended $currentArrayParam = []; // We build array param here before pushing when it's ended $stack = []; // State for stack of parentheses $stackCount = 0; // Length of stack array. Kept as variable to improve performance $stringStackState = null; // State for string support - // Loop thorough all characters for ($i = $parametersStart; $i < $paramsEnd; $i++) { $char = $filter[$i]; @@ -135,20 +136,25 @@ class V17 extends Filter ($filter[$i - 1] !== static::CHAR_BACKSLASH || $filter[$i - 2] === static::CHAR_BACKSLASH) // Must not be escaped; ) { if ($isStringStack) { - // Dont mix-up string symbols. Only allow the same as on start + // Don't mix up string symbols. Only allow the same as on start if ($char === $stringStackState) { // End of string $stringStackState = null; } - - // Either way, add symbol to builder - static::appendSymbol($isStringStack, $char, $i, $filter, $currentParam); } else { // Start of string $stringStackState = $char; - static::appendSymbol($isStringStack, $char, $i, $filter, $currentParam); } + // Either way, add symbol to builder + static::appendSymbol( + $isStringStack, + $char, + $i, + $filter, + $currentParam, + ); + continue; } @@ -174,12 +180,12 @@ class V17 extends Filter continue; } elseif ($char === static::CHAR_COMMA) { // Params separation support - // If in array stack, dont merge yet, just mark it in array param builder + // If in array stack, don't merge yet, just mark it in array param builder if ($isArrayStack) { $currentArrayParam[] = $currentParam; $currentParam = ""; } else { - // Append from parap builder. Either value, or array + // Append from param builder. Either value, or array if (empty($currentArrayParam)) { if (strlen($currentParam)) { $params[] = $currentParam; @@ -193,23 +199,28 @@ class V17 extends Filter } // Value, not relevant to syntax - static::appendSymbol($isStringStack, $char, $i, $filter, $currentParam); + static::appendSymbol( + $isStringStack, + $char, + $i, + $filter, + $currentParam, + ); } - if (strlen($currentParam)) { + if (\strlen($currentParam)) { $params[] = $currentParam; - $currentParam = ""; + $currentParam = ''; } $parsedParams = []; foreach ($params as $param) { - // If array, parse each child separatelly + // If array, parse each child separately if (\is_array($param)) { foreach ($param as $element) { $arr[] = self::parseValue($element); } - $parsedParams[] = $arr ?? []; } else { $parsedParams[] = self::parseValue($param); @@ -295,8 +306,13 @@ class V17 extends Filter * @param string $currentParam * @return void */ - private function appendSymbol(bool $isStringStack, string $char, int $index, string $filter, string &$currentParam): void - { + private function appendSymbol( + bool $isStringStack, + string $char, + int $index, + string $filter, + string &$currentParam + ): void { // Ignore spaces and commas outside of string $canBeIgnored = false; diff --git a/src/Appwrite/Utopia/Request/Filters/V20.php b/src/Appwrite/Utopia/Request/Filters/V20.php index c8622f8b7a..51f8bcf23a 100644 --- a/src/Appwrite/Utopia/Request/Filters/V20.php +++ b/src/Appwrite/Utopia/Request/Filters/V20.php @@ -2,8 +2,10 @@ namespace Appwrite\Utopia\Request\Filters; +use Appwrite\Extend\Exception; use Appwrite\Utopia\Request\Filter; use Utopia\Database\Database; +use Utopia\Database\Exception\NotFound; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -54,8 +56,8 @@ class V20 extends Filter try { $parsed = Query::parseQueries($content['queries']); - } catch (QueryException) { - return $content; + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } $selections = Query::groupByType($parsed)['selections'] ?? []; @@ -136,17 +138,28 @@ class V20 extends Filter return []; } - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - if ($database->isEmpty()) { - return []; + try { + $database = Authorization::skip(fn() => $dbForProject->getDocument( + 'databases', + $databaseId + )); + if ($database->isEmpty()) { + return []; + } + } catch (NotFound) { + throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument( - 'database_' . $database->getSequence(), - $collectionId - )); - if ($collection->isEmpty()) { - return []; + try { + $collection = Authorization::skip(fn() => $dbForProject->getDocument( + 'database_' . $database->getSequence(), + $collectionId + )); + if ($collection->isEmpty()) { + return []; + } + } catch (NotFound) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); } $attributes = $collection->getAttribute('attributes', []); From c9b39a3f28996af70e5c9b9ecca42b8c58a9c7ae Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 16 Sep 2025 18:37:52 +1200 Subject: [PATCH 13/26] Lint --- src/Appwrite/Utopia/Request/Filters/V20.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Utopia/Request/Filters/V20.php b/src/Appwrite/Utopia/Request/Filters/V20.php index 51f8bcf23a..3783a61947 100644 --- a/src/Appwrite/Utopia/Request/Filters/V20.php +++ b/src/Appwrite/Utopia/Request/Filters/V20.php @@ -139,7 +139,7 @@ class V20 extends Filter } try { - $database = Authorization::skip(fn() => $dbForProject->getDocument( + $database = Authorization::skip(fn () => $dbForProject->getDocument( 'databases', $databaseId )); @@ -151,7 +151,7 @@ class V20 extends Filter } try { - $collection = Authorization::skip(fn() => $dbForProject->getDocument( + $collection = Authorization::skip(fn () => $dbForProject->getDocument( 'database_' . $database->getSequence(), $collectionId )); From 832687dd091c7cc33482e946f5572652b4ebea67 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 16 Sep 2025 23:42:07 +1200 Subject: [PATCH 14/26] Catch query exception on bucket/file list --- app/controllers/api/storage.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8cfeb5da3b..8bc383cabd 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -225,6 +225,8 @@ App::get('/v1/storage/buckets') $total = $dbForProject->count('buckets', $filterQueries, APP_LIMIT_COUNT); } catch (OrderException $e) { throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } $response->dynamic(new Document([ 'buckets' => $buckets, @@ -853,6 +855,8 @@ App::get('/v1/storage/buckets/:bucketId/files') throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } catch (OrderException $e) { throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } $response->dynamic(new Document([ From f03a62d43dd8f9ee84e6d1a4004094b7ed930833 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 17 Sep 2025 01:33:36 +1200 Subject: [PATCH 15/26] Update database --- composer.lock | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/composer.lock b/composer.lock index e454de0313..0ed0d05dc5 100644 --- a/composer.lock +++ b/composer.lock @@ -756,24 +756,21 @@ }, { "name": "google/protobuf", - "version": "v4.32.0", + "version": "v4.32.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646" + "reference": "c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646", - "reference": "9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb", + "reference": "c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb", "shasum": "" }, "require": { "php": ">=8.1.0" }, - "provide": { - "ext-protobuf": "*" - }, "require-dev": { "phpunit/phpunit": ">=5.0.0 <8.5.27" }, @@ -797,9 +794,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.32.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.32.1" }, - "time": "2025-08-14T20:00:33+00:00" + "time": "2025-09-14T05:14:52+00:00" }, { "name": "league/csv", @@ -3638,16 +3635,16 @@ }, { "name": "utopia-php/database", - "version": "1.4.8", + "version": "1.4.9", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "dbecdf89fde33a5f81ec19f4f97fe0c3715dc83a" + "reference": "066e2bda7b728bb843776db3640737d7350ba035" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/dbecdf89fde33a5f81ec19f4f97fe0c3715dc83a", - "reference": "dbecdf89fde33a5f81ec19f4f97fe0c3715dc83a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/066e2bda7b728bb843776db3640737d7350ba035", + "reference": "066e2bda7b728bb843776db3640737d7350ba035", "shasum": "" }, "require": { @@ -3688,9 +3685,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/1.4.8" + "source": "https://github.com/utopia-php/database/tree/1.4.9" }, - "time": "2025-09-12T03:35:59+00:00" + "time": "2025-09-16T13:31:52+00:00" }, { "name": "utopia-php/detector", @@ -6236,16 +6233,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.26", + "version": "9.6.27", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "a0139ea157533454f611038326f3020b3051f129" + "reference": "0a9aa4440b6a9528cf360071502628d717af3e0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/a0139ea157533454f611038326f3020b3051f129", - "reference": "a0139ea157533454f611038326f3020b3051f129", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0a9aa4440b6a9528cf360071502628d717af3e0a", + "reference": "0a9aa4440b6a9528cf360071502628d717af3e0a", "shasum": "" }, "require": { @@ -6319,7 +6316,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.26" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.27" }, "funding": [ { @@ -6343,7 +6340,7 @@ "type": "tidelift" } ], - "time": "2025-09-11T06:17:45+00:00" + "time": "2025-09-14T06:18:03+00:00" }, { "name": "psr/cache", From 7d1a1d3ef479160b05fb24df665ad3ade5e7d35d Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 17 Sep 2025 08:45:23 +0300 Subject: [PATCH 16/26] use $tenant --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 9d998d05bc..bd34d53b00 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -504,7 +504,7 @@ class StatsUsage extends Action usort($this->statDocuments, function ($a, $b) { // Tenant ASC - $cmp = $a['_tenant'] <=> $b['_tenant']; + $cmp = $a['$tenant'] <=> $b['$tenant']; if ($cmp !== 0) { return $cmp; } From 481578047690d2d49fc699c4dd7eff96bebffd9f Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 17 Sep 2025 18:01:29 +0530 Subject: [PATCH 17/26] Add spatial column validation during required mode and tests for existing data in databases --- .../Collections/Attributes/Action.php | 9 +- .../Databases/Legacy/DatabasesBase.php | 130 +++++++++++++++++ .../Databases/TablesDB/DatabasesBase.php | 138 +++++++++++++++++- 3 files changed, 272 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php index f3903d91a7..f8d49734eb 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php @@ -366,13 +366,20 @@ abstract class Action extends UtopiaAction 'filters' => $filters, 'options' => $options, ]); - + if (!$dbForProject->getAdapter()->getSupportForSpatialIndexNull() && in_array($attribute->getAttribute('type'), Database::SPATIAL_TYPES) && $attribute->getAttribute('required')) { + $existingDataPresent = $dbForProject->findOne('database_' . $db->getSequence() . '_collection_' . $collection->getSequence()); + if (count($existingDataPresent)) { + throw new StructureException('Failed to add required spatial column: existing rows present. Make the column optional.'); + } + } $dbForProject->checkAttribute($collection, $attribute); $attribute = $dbForProject->createDocument('attributes', $attribute); } catch (DuplicateException) { throw new Exception($this->getDuplicateException()); } catch (LimitException) { throw new Exception($this->getLimitException()); + } catch (StructureException $e) { + throw new Exception($this->getInvalidStructureException(), $e->getMessage()); } catch (Throwable $e) { $dbForProject->purgeCachedDocument('database_' . $db->getSequence(), $collectionId); $dbForProject->purgeCachedCollection('database_' . $db->getSequence() . '_collection_' . $collection->getSequence()); diff --git a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php index 8d27aa7230..89c0cb6de0 100644 --- a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php +++ b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php @@ -7798,4 +7798,134 @@ trait DatabasesBase 'x-appwrite-key' => $this->getProject()['apiKey'] ])); } + + public function testSpatialColCreateOnExistingData(): void + { + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Spatial Distance Meters Database' + ]); + + $databaseId = $database['body']['$id']; + + $colId = ID::unique(); + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => $colId, + 'name' => 'spatial-test', + 'documentSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + + $description = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'description', + 'size' => 512, + 'required' => false, + 'default' => '', + ]); + + $this->assertEquals(202, $description['headers']['status-code']); + sleep(2); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'description' => 'description' + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ] + ]); + $this->assertEquals(201, $document['headers']['status-code']); + + $point = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/point', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'loc', + 'required' => true, + ]); + + $this->assertEquals(400, $point['headers']['status-code']); + + $point = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/point', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'loc', + 'required' => false, + 'default' => null + ]); + + $this->assertEquals(202, $point['headers']['status-code']); + + $line = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/line', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'route', + 'required' => true, + ]); + + $this->assertEquals(400, $line['headers']['status-code']); + + $line = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/line', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'route', + 'required' => false, + 'default' => null + ]); + + $this->assertEquals(202, $line['headers']['status-code']); + + $poly = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/polygon', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'area', + 'required' => true, + ]); + + $this->assertEquals(400, $poly['headers']['status-code']); + + $poly = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/polygon', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'area', + 'required' => false, + 'default' => null + ]); + + $this->assertEquals(202, $poly['headers']['status-code']); + } } diff --git a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php index eba7ec96a7..70d4420828 100644 --- a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php @@ -3062,7 +3062,7 @@ trait DatabasesBase public function testInvalidRowStructure(): void { - $database = $this->client->call(Client::METHOD_POST, '/databases', array_merge([ + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -3735,7 +3735,7 @@ trait DatabasesBase public function testEnforceTableAndRowPermissions(): void { - $database = $this->client->call(Client::METHOD_POST, '/databases', array_merge([ + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -3928,7 +3928,7 @@ trait DatabasesBase public function testEnforceTablePermissions(): void { - $database = $this->client->call(Client::METHOD_POST, '/databases', array_merge([ + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -4284,7 +4284,7 @@ trait DatabasesBase public function testUpdatePermissionsWithEmptyPayload(): array { // Create Database - $database = $this->client->call(Client::METHOD_POST, '/databases', array_merge([ + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -8828,4 +8828,134 @@ trait DatabasesBase $this->client->call(Client::METHOD_DELETE, "/tablesdb/{$databaseId}", $headers); } + public function testSpatialColCreateOnExistingData(): void + { + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Spatial Distance Meters Database' + ]); + + $databaseId = $database['body']['$id']; + + $tableId = ID::unique(); + $table = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'tableId' => $tableId, + 'name' => 'spatial-test', + 'rowSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $table['headers']['status-code']); + + $description = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'description', + 'size' => 512, + 'required' => false, + 'default' => '', + ]); + + $this->assertEquals(202, $description['headers']['status-code']); + sleep(2); + + $row = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'rowId' => ID::unique(), + 'data' => [ + 'description' => 'description' + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ] + ]); + $this->assertEquals(201, $row['headers']['status-code']); + + $point = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/point', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'loc', + 'required' => true, + ]); + + $this->assertEquals(400, $point['headers']['status-code']); + + $point = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/point', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'loc', + 'required' => false, + 'default' => null + ]); + + $this->assertEquals(202, $point['headers']['status-code']); + + $line = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/line', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'route', + 'required' => true, + ]); + + $this->assertEquals(400, $line['headers']['status-code']); + + $line = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/line', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'route', + 'required' => false, + 'default' => null + ]); + + $this->assertEquals(202, $line['headers']['status-code']); + + $poly = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/polygon', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'area', + 'required' => true, + ]); + + $this->assertEquals(400, $poly['headers']['status-code']); + + $poly = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/polygon', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'area', + 'required' => false, + 'default' => null + ]); + + $this->assertEquals(202, $poly['headers']['status-code']); + } + } From 5b09ebe54b494cd6479612b2ff850f176fd4f13c Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 17 Sep 2025 19:46:06 +0530 Subject: [PATCH 18/26] added cases for with default values --- .../Databases/Legacy/DatabasesBase.php | 100 ++++++++++++++++++ .../Databases/TablesDB/DatabasesBase.php | 100 ++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php index 89c0cb6de0..1dab5076ec 100644 --- a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php +++ b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php @@ -7928,4 +7928,104 @@ trait DatabasesBase $this->assertEquals(202, $poly['headers']['status-code']); } + + public function testSpatialColCreateOnExistingDataWithDefaults(): void + { + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Spatial With Defaults Database' + ]); + + $databaseId = $database['body']['$id']; + + $colId = ID::unique(); + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => $colId, + 'name' => 'spatial-test-defaults', + 'documentSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + + $description = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'description', + 'size' => 512, + 'required' => false, + 'default' => '', + ]); + + $this->assertEquals(202, $description['headers']['status-code']); + sleep(2); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'description' => 'description' + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ] + ]); + $this->assertEquals(201, $document['headers']['status-code']); + + // Test point with default value + $point = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/point', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'loc', + 'required' => false, + 'default' => [0.0, 0.0] + ]); + + $this->assertEquals(202, $point['headers']['status-code']); + + // Test line with default value + $line = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/line', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'route', + 'required' => false, + 'default' => [[0.0, 0.0], [1.0, 1.0]] + ]); + + $this->assertEquals(202, $line['headers']['status-code']); + + // Test polygon with default value + $poly = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/attributes/polygon', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'area', + 'required' => false, + 'default' => [[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0]]] + ]); + + $this->assertEquals(202, $poly['headers']['status-code']); + } } diff --git a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php index 70d4420828..ebfa0132e6 100644 --- a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php @@ -8958,4 +8958,104 @@ trait DatabasesBase $this->assertEquals(202, $poly['headers']['status-code']); } + public function testSpatialColCreateOnExistingDataWithDefaults(): void + { + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Spatial With Defaults Database' + ]); + + $databaseId = $database['body']['$id']; + + $tableId = ID::unique(); + $table = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'tableId' => $tableId, + 'name' => 'spatial-test-defaults', + 'rowSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $table['headers']['status-code']); + + $description = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'description', + 'size' => 512, + 'required' => false, + 'default' => '', + ]); + + $this->assertEquals(202, $description['headers']['status-code']); + sleep(2); + + $row = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'rowId' => ID::unique(), + 'data' => [ + 'description' => 'description' + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ] + ]); + $this->assertEquals(201, $row['headers']['status-code']); + + // Test point with default value + $point = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/point', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'loc', + 'required' => false, + 'default' => [0.0, 0.0] + ]); + + $this->assertEquals(202, $point['headers']['status-code']); + + // Test line with default value + $line = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/line', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'route', + 'required' => false, + 'default' => [[0.0, 0.0], [1.0, 1.0]] + ]); + + $this->assertEquals(202, $line['headers']['status-code']); + + // Test polygon with default value + $poly = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/polygon', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'area', + 'required' => false, + 'default' => [[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0]]] + ]); + + $this->assertEquals(202, $poly['headers']['status-code']); + } + } From c5e9ec396929e6683c7f55771046b88604ba2c9e Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 17 Sep 2025 20:00:10 +0530 Subject: [PATCH 19/26] added cases for checking default added or not --- .../Databases/Legacy/DatabasesBase.php | 35 +++++++++++++++++++ .../Databases/TablesDB/DatabasesBase.php | 35 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php index 1dab5076ec..c9a293dcb0 100644 --- a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php +++ b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php @@ -8027,5 +8027,40 @@ trait DatabasesBase ]); $this->assertEquals(202, $poly['headers']['status-code']); + + // Wait for attributes to be available + sleep(2); + + // Create a new document without spatial data to test default values + $newDocument = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $colId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'description' => 'test default values' + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ] + ]); + $this->assertEquals(201, $newDocument['headers']['status-code']); + + $newDocumentId = $newDocument['body']['$id']; + + // Fetch the document to verify default values are applied + $fetchedDocument = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $colId . '/documents/' . $newDocumentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $fetchedDocument['headers']['status-code']); + + // Verify default values are applied + $this->assertEquals([0.0, 0.0], $fetchedDocument['body']['loc']); + $this->assertEquals([[0.0, 0.0], [1.0, 1.0]], $fetchedDocument['body']['route']); + $this->assertEquals([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0]]], $fetchedDocument['body']['area']); } } diff --git a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php index ebfa0132e6..7be23ca136 100644 --- a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php @@ -9056,6 +9056,41 @@ trait DatabasesBase ]); $this->assertEquals(202, $poly['headers']['status-code']); + + // Wait for columns to be available + sleep(2); + + // Create a new row without spatial data to test default values + $newRow = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'rowId' => ID::unique(), + 'data' => [ + 'description' => 'test default values' + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ] + ]); + $this->assertEquals(201, $newRow['headers']['status-code']); + + $newRowId = $newRow['body']['$id']; + + // Fetch the row to verify default values are applied + $fetchedRow = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows/' . $newRowId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $fetchedRow['headers']['status-code']); + + // Verify default values are applied + $this->assertEquals([0.0, 0.0], $fetchedRow['body']['loc']); + $this->assertEquals([[0.0, 0.0], [1.0, 1.0]], $fetchedRow['body']['route']); + $this->assertEquals([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0]]], $fetchedRow['body']['area']); } } From 52ceddcd5ae1dad4ccf364cb931d0d9634275572 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 17 Sep 2025 20:02:36 +0530 Subject: [PATCH 20/26] linting --- tests/e2e/Services/Databases/Legacy/DatabasesBase.php | 8 ++++---- tests/e2e/Services/Databases/TablesDB/DatabasesBase.php | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php index c9a293dcb0..8b4f78d3ea 100644 --- a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php +++ b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php @@ -8027,7 +8027,7 @@ trait DatabasesBase ]); $this->assertEquals(202, $poly['headers']['status-code']); - + // Wait for attributes to be available sleep(2); @@ -8047,7 +8047,7 @@ trait DatabasesBase ] ]); $this->assertEquals(201, $newDocument['headers']['status-code']); - + $newDocumentId = $newDocument['body']['$id']; // Fetch the document to verify default values are applied @@ -8055,9 +8055,9 @@ trait DatabasesBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); - + $this->assertEquals(200, $fetchedDocument['headers']['status-code']); - + // Verify default values are applied $this->assertEquals([0.0, 0.0], $fetchedDocument['body']['loc']); $this->assertEquals([[0.0, 0.0], [1.0, 1.0]], $fetchedDocument['body']['route']); diff --git a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php index 7be23ca136..387a7ec3f0 100644 --- a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php @@ -9056,7 +9056,7 @@ trait DatabasesBase ]); $this->assertEquals(202, $poly['headers']['status-code']); - + // Wait for columns to be available sleep(2); @@ -9076,7 +9076,7 @@ trait DatabasesBase ] ]); $this->assertEquals(201, $newRow['headers']['status-code']); - + $newRowId = $newRow['body']['$id']; // Fetch the row to verify default values are applied @@ -9084,9 +9084,9 @@ trait DatabasesBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); - + $this->assertEquals(200, $fetchedRow['headers']['status-code']); - + // Verify default values are applied $this->assertEquals([0.0, 0.0], $fetchedRow['body']['loc']); $this->assertEquals([[0.0, 0.0], [1.0, 1.0]], $fetchedRow['body']['route']); From 0e2c5d93f5951a7f20cb0343593edeeb8791dd9a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 18 Sep 2025 14:54:07 +1200 Subject: [PATCH 21/26] Update check --- composer.lock | 41 +++++++++---------- .../Collections/Attributes/Action.php | 21 ++++++---- .../Collections/Documents/Action.php | 2 +- .../Collections/Documents/Bulk/Update.php | 2 +- .../Collections/Documents/Bulk/Upsert.php | 2 +- .../Collections/Documents/Create.php | 2 +- .../Collections/Documents/Update.php | 2 +- .../Collections/Documents/Upsert.php | 2 +- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/composer.lock b/composer.lock index 0ed0d05dc5..d8c17fbc91 100644 --- a/composer.lock +++ b/composer.lock @@ -1352,16 +1352,16 @@ }, { "name": "open-telemetry/gen-otlp-protobuf", - "version": "1.5.0", + "version": "1.8.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/gen-otlp-protobuf.git", - "reference": "585bafddd4ae6565de154610b10a787a455c9ba0" + "reference": "673af5b06545b513466081884b47ef15a536edde" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/585bafddd4ae6565de154610b10a787a455c9ba0", - "reference": "585bafddd4ae6565de154610b10a787a455c9ba0", + "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/673af5b06545b513466081884b47ef15a536edde", + "reference": "673af5b06545b513466081884b47ef15a536edde", "shasum": "" }, "require": { @@ -1411,7 +1411,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-15T23:07:07+00:00" + "time": "2025-09-17T23:10:12+00:00" }, { "name": "open-telemetry/sdk", @@ -3635,16 +3635,16 @@ }, { "name": "utopia-php/database", - "version": "1.4.9", + "version": "1.4.10", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "066e2bda7b728bb843776db3640737d7350ba035" + "reference": "5514bb7346e75996d061d08040248fe842f73785" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/066e2bda7b728bb843776db3640737d7350ba035", - "reference": "066e2bda7b728bb843776db3640737d7350ba035", + "url": "https://api.github.com/repos/utopia-php/database/zipball/5514bb7346e75996d061d08040248fe842f73785", + "reference": "5514bb7346e75996d061d08040248fe842f73785", "shasum": "" }, "require": { @@ -3685,9 +3685,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/1.4.9" + "source": "https://github.com/utopia-php/database/tree/1.4.10" }, - "time": "2025-09-16T13:31:52+00:00" + "time": "2025-09-18T02:42:25+00:00" }, { "name": "utopia-php/detector", @@ -5278,16 +5278,16 @@ }, { "name": "laravel/pint", - "version": "v1.24.0", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a" + "reference": "595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/0345f3b05f136801af8c339f9d16ef29e6b4df8a", - "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a", + "url": "https://api.github.com/repos/laravel/pint/zipball/595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96", + "reference": "595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96", "shasum": "" }, "require": { @@ -5298,9 +5298,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.82.2", - "illuminate/view": "^11.45.1", - "larastan/larastan": "^3.5.0", + "friendsofphp/php-cs-fixer": "^3.87.2", + "illuminate/view": "^11.46.0", + "larastan/larastan": "^3.7.1", "laravel-zero/framework": "^11.45.0", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3.1", @@ -5311,9 +5311,6 @@ ], "type": "project", "autoload": { - "files": [ - "overrides/Runner/Parallel/ProcessFactory.php" - ], "psr-4": { "App\\": "app/", "Database\\Seeders\\": "database/seeders/", @@ -5343,7 +5340,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-07-10T18:09:32+00:00" + "time": "2025-09-17T01:36:44+00:00" }, { "name": "matthiasmullie/minify", diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php index f8d49734eb..22a90d2653 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Attributes/Action.php @@ -122,7 +122,7 @@ abstract class Action extends UtopiaAction /** * Get the correct invalid structure message. */ - protected function getInvalidStructureException(): string + protected function getStructureException(): string { return $this->isCollectionsAPI() ? Exception::DOCUMENT_INVALID_STRUCTURE @@ -366,9 +366,16 @@ abstract class Action extends UtopiaAction 'filters' => $filters, 'options' => $options, ]); - if (!$dbForProject->getAdapter()->getSupportForSpatialIndexNull() && in_array($attribute->getAttribute('type'), Database::SPATIAL_TYPES) && $attribute->getAttribute('required')) { - $existingDataPresent = $dbForProject->findOne('database_' . $db->getSequence() . '_collection_' . $collection->getSequence()); - if (count($existingDataPresent)) { + if ( + !$dbForProject->getAdapter()->getSupportForSpatialIndexNull() && + \in_array($attribute->getAttribute('type'), Database::SPATIAL_TYPES) && + $attribute->getAttribute('required') + ) { + $hasData = !Authorization::skip(fn () => $dbForProject + ->findOne('database_' . $db->getSequence() . '_collection_' . $collection->getSequence())) + ->isEmpty(); + + if ($hasData) { throw new StructureException('Failed to add required spatial column: existing rows present. Make the column optional.'); } } @@ -379,7 +386,7 @@ abstract class Action extends UtopiaAction } catch (LimitException) { throw new Exception($this->getLimitException()); } catch (StructureException $e) { - throw new Exception($this->getInvalidStructureException(), $e->getMessage()); + throw new Exception($this->getStructureException(), $e->getMessage()); } catch (Throwable $e) { $dbForProject->purgeCachedDocument('database_' . $db->getSequence(), $collectionId); $dbForProject->purgeCachedCollection('database_' . $db->getSequence() . '_collection_' . $collection->getSequence()); @@ -423,7 +430,7 @@ abstract class Action extends UtopiaAction } catch (LimitException) { throw new Exception($this->getLimitException()); } catch (StructureException) { - throw new Exception($this->getInvalidStructureException()); + throw new Exception($this->getStructureException()); } catch (Throwable $e) { $dbForProject->deleteDocument('attributes', $attribute->getId()); throw $e; @@ -587,7 +594,7 @@ abstract class Action extends UtopiaAction } catch (RelationshipException $e) { throw new Exception(Exception::RELATIONSHIP_VALUE_INVALID, $e->getMessage()); } catch (StructureException $e) { - throw new Exception($this->getInvalidStructureException(), $e->getMessage()); + throw new Exception($this->getStructureException(), $e->getMessage()); } if ($primaryDocumentOptions['twoWay']) { diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php index d1d0738990..e05e588201 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php @@ -160,7 +160,7 @@ abstract class Action extends AppwriteAction /** * Get the correct invalid structure message. */ - protected function getInvalidStructureException(): string + protected function getStructureException(): string { return $this->isCollectionsAPI() ? Exception::DOCUMENT_INVALID_STRUCTURE diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php index 0f0ae14020..baab45cdd3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php @@ -149,7 +149,7 @@ class Update extends Action } catch (RelationshipException $e) { throw new Exception(Exception::RELATIONSHIP_VALUE_INVALID, $e->getMessage()); } catch (StructureException $e) { - throw new Exception($this->getInvalidStructureException(), $e->getMessage()); + throw new Exception($this->getStructureException(), $e->getMessage()); } foreach ($documents as $document) { diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index 395e3d757b..4ce3990a38 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -130,7 +130,7 @@ class Upsert extends Action } catch (RelationshipException $e) { throw new Exception(Exception::RELATIONSHIP_VALUE_INVALID, $e->getMessage()); } catch (StructureException $e) { - throw new Exception($this->getInvalidStructureException(), $e->getMessage()); + throw new Exception($this->getStructureException(), $e->getMessage()); } foreach ($upserted as $document) { diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php index c03daabd4f..a8af1eda86 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php @@ -375,7 +375,7 @@ class Create extends Action } catch (RelationshipException $e) { throw new Exception(Exception::RELATIONSHIP_VALUE_INVALID, $e->getMessage()); } catch (StructureException $e) { - throw new Exception($this->getInvalidStructureException(), $e->getMessage()); + throw new Exception($this->getStructureException(), $e->getMessage()); } $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php index 8382bdd5e9..e510aeb089 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php @@ -247,7 +247,7 @@ class Update extends Action } catch (RelationshipException $e) { throw new Exception(Exception::RELATIONSHIP_VALUE_INVALID, $e->getMessage()); } catch (StructureException $e) { - throw new Exception($this->getInvalidStructureException(), $e->getMessage()); + throw new Exception($this->getStructureException(), $e->getMessage()); } $collectionsCache = []; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php index 54b1cad950..3ac5e704e3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php @@ -258,7 +258,7 @@ class Upsert extends Action } catch (RelationshipException $e) { throw new Exception(Exception::RELATIONSHIP_VALUE_INVALID, $e->getMessage()); } catch (StructureException $e) { - throw new Exception($this->getInvalidStructureException(), $e->getMessage()); + throw new Exception($this->getStructureException(), $e->getMessage()); } $collectionsCache = []; From a7e9e58312afbb93b49eea2fee1a61c05b2767dc Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 18 Sep 2025 10:57:47 +0300 Subject: [PATCH 22/26] Add order by --- app/init/database/filters.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/init/database/filters.php b/app/init/database/filters.php index 33f5d8077a..8261cdbc30 100644 --- a/app/init/database/filters.php +++ b/app/init/database/filters.php @@ -255,6 +255,8 @@ Database::addFilter( ->find('variables', [ Query::equal('resourceInternalId', [$document->getSequence()]), Query::equal('resourceType', $resourceType), + Query::orderAsc('resourceType'), + Query::orderAsc('$sequence'), Query::limit(APP_LIMIT_SUBQUERY), ]); } From 4da7f04a5c81d8835857cfeacefafb4feb0a0dc4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 18 Sep 2025 10:58:27 +0300 Subject: [PATCH 23/26] Add order by --- app/init/database/filters.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init/database/filters.php b/app/init/database/filters.php index 8261cdbc30..c4cfd1ac81 100644 --- a/app/init/database/filters.php +++ b/app/init/database/filters.php @@ -256,7 +256,7 @@ Database::addFilter( Query::equal('resourceInternalId', [$document->getSequence()]), Query::equal('resourceType', $resourceType), Query::orderAsc('resourceType'), - Query::orderAsc('$sequence'), + Query::orderAsc(), Query::limit(APP_LIMIT_SUBQUERY), ]); } From d9b657796b6f84d239802c44040ade2e3bb92a49 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 18 Sep 2025 17:49:38 +0300 Subject: [PATCH 24/26] bump database 1.5.0 --- composer.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/composer.lock b/composer.lock index d8c17fbc91..98089f9ed9 100644 --- a/composer.lock +++ b/composer.lock @@ -3635,16 +3635,16 @@ }, { "name": "utopia-php/database", - "version": "1.4.10", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "5514bb7346e75996d061d08040248fe842f73785" + "reference": "24c4519b4ac32aee13af31dddd984db2a3b34980" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/5514bb7346e75996d061d08040248fe842f73785", - "reference": "5514bb7346e75996d061d08040248fe842f73785", + "url": "https://api.github.com/repos/utopia-php/database/zipball/24c4519b4ac32aee13af31dddd984db2a3b34980", + "reference": "24c4519b4ac32aee13af31dddd984db2a3b34980", "shasum": "" }, "require": { @@ -3685,9 +3685,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/1.4.10" + "source": "https://github.com/utopia-php/database/tree/1.5.0" }, - "time": "2025-09-18T02:42:25+00:00" + "time": "2025-09-18T14:42:01+00:00" }, { "name": "utopia-php/detector", @@ -5004,16 +5004,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.3.4", + "version": "1.3.5", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "d3b420dced42f1eec1f6d0aa98b7bbf8de4042ac" + "reference": "6fda9e58b37c9872c1a2a424e5467de8de1bc567" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d3b420dced42f1eec1f6d0aa98b7bbf8de4042ac", - "reference": "d3b420dced42f1eec1f6d0aa98b7bbf8de4042ac", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6fda9e58b37c9872c1a2a424e5467de8de1bc567", + "reference": "6fda9e58b37c9872c1a2a424e5467de8de1bc567", "shasum": "" }, "require": { @@ -5049,9 +5049,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.3.4" + "source": "https://github.com/appwrite/sdk-generator/tree/1.3.5" }, - "time": "2025-09-08T11:56:04+00:00" + "time": "2025-09-15T04:19:40+00:00" }, { "name": "doctrine/annotations", @@ -8506,7 +8506,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From c1782f6f58998cf5a6f3d5fd270ed697f98999b1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 19 Sep 2025 15:28:41 +1200 Subject: [PATCH 25/26] Don't remove required attributes --- .../Databases/Collections/Documents/XList.php | 37 ------------------- .../Databases/Legacy/DatabasesBase.php | 12 +++--- .../Databases/TablesDB/DatabasesBase.php | 12 +++--- 3 files changed, 12 insertions(+), 49 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php index 9c8405cf18..546cbeddd4 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php @@ -158,43 +158,6 @@ class XList extends Action ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); - // Check if the SELECT query includes the removable attributes - $hasWildcard = false; - $hasSelectQueries = !empty($selectQueries); - $requestedAttributes = []; - - if ($hasSelectQueries) { - foreach ($selectQueries as $query) { - if ($query->getMethod() !== Query::TYPE_SELECT) { - continue; - } - - $values = $query->getValues(); - if (\in_array('*', $values, true)) { - $hasWildcard = true; - break; - } - - // Check which removable attributes are explicitly requested - foreach ($this->removableAttributes['*'] as $attribute) { - if (\in_array($attribute, $values, true)) { - $requestedAttributes[$attribute] = true; - } - } - } - - if (!$hasWildcard) { - foreach ($documents as $document) { - // Remove attributes that are not explicitly requested - foreach ($this->removableAttributes['*'] as $attribute) { - if (!isset($requestedAttributes[$attribute])) { - $document->removeAttribute($attribute); - } - } - } - } - } - $response->dynamic(new Document([ 'total' => $total, // rows or documents diff --git a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php index 8b4f78d3ea..a432bc0acd 100644 --- a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php +++ b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php @@ -5270,8 +5270,8 @@ trait DatabasesBase $this->assertEquals(2, count($response['body']['documents'])); $this->assertEquals(null, $response['body']['documents'][0]['fullName']); $this->assertArrayNotHasKey("libraries", $response['body']['documents'][0]); - $this->assertArrayNotHasKey('$databaseId', $response['body']['documents'][0]); - $this->assertArrayNotHasKey('$collectionId', $response['body']['documents'][0]); + $this->assertArrayHasKey('$databaseId', $response['body']['documents'][0]); + $this->assertArrayHasKey('$collectionId', $response['body']['documents'][0]); } /** @@ -5291,8 +5291,8 @@ trait DatabasesBase $this->assertEquals(200, $response['headers']['status-code']); $this->assertArrayNotHasKey('libraries', $response['body']['documents'][0]); - $this->assertArrayNotHasKey('$databaseId', $response['body']['documents'][0]); - $this->assertArrayNotHasKey('$collectionId', $response['body']['documents'][0]); + $this->assertArrayHasKey('$databaseId', $response['body']['documents'][0]); + $this->assertArrayHasKey('$collectionId', $response['body']['documents'][0]); $response = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/collections/' . $data['personCollection'] . '/documents', array_merge([ 'content-type' => 'application/json', @@ -5305,8 +5305,8 @@ trait DatabasesBase $document = $response['body']['documents'][0]; $this->assertEquals(200, $response['headers']['status-code']); $this->assertArrayHasKey('libraries', $document); - $this->assertArrayNotHasKey('$databaseId', $document); - $this->assertArrayNotHasKey('$collectionId', $document); + $this->assertArrayHasKey('$databaseId', $document); + $this->assertArrayHasKey('$collectionId', $document); $response = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/collections/' . $data['personCollection'] . '/documents/' . $document['$id'], array_merge([ 'content-type' => 'application/json', diff --git a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php index 387a7ec3f0..336193f5d9 100644 --- a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php @@ -5197,8 +5197,8 @@ trait DatabasesBase $this->assertEquals(2, count($response['body']['rows'])); $this->assertEquals(null, $response['body']['rows'][0]['fullName']); $this->assertArrayNotHasKey("libraries", $response['body']['rows'][0]); - $this->assertArrayNotHasKey('$databaseId', $response['body']['rows'][0]); - $this->assertArrayNotHasKey('$tableId', $response['body']['rows'][0]); + $this->assertArrayHasKey('$databaseId', $response['body']['rows'][0]); + $this->assertArrayHasKey('$tableId', $response['body']['rows'][0]); } /** @@ -5218,8 +5218,8 @@ trait DatabasesBase $this->assertEquals(200, $response['headers']['status-code']); $this->assertArrayNotHasKey('libraries', $response['body']['rows'][0]); - $this->assertArrayNotHasKey('$databaseId', $response['body']['rows'][0]); - $this->assertArrayNotHasKey('$tableId', $response['body']['rows'][0]); + $this->assertArrayHasKey('$databaseId', $response['body']['rows'][0]); + $this->assertArrayHasKey('$tableId', $response['body']['rows'][0]); $response = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $data['databaseId'] . '/tables/' . $data['personCollection'] . '/rows', array_merge([ 'content-type' => 'application/json', @@ -5232,8 +5232,8 @@ trait DatabasesBase $row = $response['body']['rows'][0]; $this->assertEquals(200, $response['headers']['status-code']); $this->assertArrayHasKey('libraries', $row); - $this->assertArrayNotHasKey('$databaseId', $row); - $this->assertArrayNotHasKey('$tableId', $row); + $this->assertArrayHasKey('$databaseId', $row); + $this->assertArrayHasKey('$tableId', $row); $response = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $data['databaseId'] . '/tables/' . $data['personCollection'] . '/rows/' . $row['$id'], array_merge([ 'content-type' => 'application/json', From f944ad350ec8e356523b7651058f6ea1259c3c87 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 19 Sep 2025 17:32:34 +1200 Subject: [PATCH 26/26] Catch query exception on bulk update/delete --- .../Http/Databases/Collections/Documents/Bulk/Delete.php | 2 ++ .../Http/Databases/Collections/Documents/Bulk/Update.php | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php index fc4e2a8a91..3467a9d11c 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php @@ -124,6 +124,8 @@ class Delete extends Action throw new Exception($this->getConflictException()); } catch (RestrictedException) { throw new Exception($this->getRestrictedException()); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } foreach ($documents as $document) { diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php index baab45cdd3..65bd255d32 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php @@ -150,6 +150,8 @@ class Update extends Action throw new Exception(Exception::RELATIONSHIP_VALUE_INVALID, $e->getMessage()); } catch (StructureException $e) { throw new Exception($this->getStructureException(), $e->getMessage()); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } foreach ($documents as $document) {