Merge pull request #10020 from appwrite/optimize-process-documents

Improve `processDocuments`
2026-05-23 17:08:45 +00:00 · 2025-06-18 13:06:18 -04:00 · 2025-06-18 13:06:18 -04:00 · da83470f4d
commit da83470f4d
parent 1337b759de d69189bd55
7 changed files with 142 additions and 243 deletions
--- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php
+++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php
@ -4,6 +4,9 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen

 use Appwrite\Extend\Exception;
 use Appwrite\Platform\Modules\Databases\Context;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Action as UtopiaAction;

 abstract class Action extends UtopiaAction
@ -193,4 +196,93 @@ abstract class Action extends UtopiaAction
    {
        return $this->isCollectionsAPI() ? 'collection' : 'table';
    }
+
+    /**
+     * Resolves relationships in a document and attaches metadata.
+     */
+    final protected function processDocument(
+        /* database */
+        Document $database,
+        Document $collection,
+        Document $document,
+        Database $dbForProject,
+
+        /* options */
+        array &$collectionsCache,
+        ?int &$operations = null,
+    ): bool {
+
+        if ($operations !== null && $document->isEmpty()) {
+            return false;
+        }
+
+        if ($operations !== null) {
+            $operations++;
+        }
+
+        $collectionId = $collection->getId();
+        $document->removeAttribute('$collection');
+        $document->setAttribute('$databaseId', $database->getId());
+        $document->setAttribute('$collectionId', $collectionId);
+
+        $relationships = $collectionsCache[$collectionId] ??= \array_filter(
+            $collection->getAttribute('attributes', []),
+            fn ($attr) => $attr->getAttribute('type') === Database::VAR_RELATIONSHIP
+        );
+
+        foreach ($relationships as $relationship) {
+            $key = $relationship->getAttribute('key');
+            $related = $document->getAttribute($key);
+
+            if (empty($related)) {
+                if (\in_array(\gettype($related), ['array', 'object']) && $operations !== null) {
+                    $operations++;
+                }
+                continue;
+            }
+
+            $relations = \is_array($related) ? $related : [$related];
+            $relatedCollectionId = $relationship->getAttribute('relatedCollection');
+
+            if (!isset($collectionsCache[$relatedCollectionId])) {
+                $relatedCollectionDoc = Authorization::skip(
+                    fn () => $dbForProject->getDocument(
+                        'database_' . $database->getSequence(),
+                        $relatedCollectionId
+                    )
+                );
+
+                $collectionsCache[$relatedCollectionId] = \array_filter(
+                    $relatedCollectionDoc->getAttribute('attributes', []),
+                    fn ($attr) => $attr->getAttribute('type') === Database::VAR_RELATIONSHIP
+                );
+            }
+
+            foreach ($relations as $relation) {
+                if ($relation instanceof Document) {
+                    $relatedCollection = new Document([
+                        '$id' => $relatedCollectionId,
+                        'attributes' => $collectionsCache[$relatedCollectionId],
+                    ]);
+
+                    $this->processDocument(
+                        database: $database,
+                        collection: $relatedCollection,
+                        document: $relation,
+                        dbForProject: $dbForProject,
+                        collectionsCache: $collectionsCache,
+                        operations: $operations
+                    );
+                }
+            }
+
+            if (\is_array($related)) {
+                $document->setAttribute($relationship->getAttribute('key'), \array_values($relations));
+            } elseif (empty($relations)) {
+                $document->setAttribute($relationship->getAttribute('key'), null);
+            }
+        }
+
+        return true;
+    }
 }
--- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php
+++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php
@ -373,42 +373,15 @@ class Create extends Action
            ->setParam('tableId', $collection->getId())
            ->setContext($this->getCollectionsEventsContext(), $collection);

-        // Add $collectionId and $databaseId for all documents
-        $processDocument = function (Document $table, Document $document) use (&$processDocument, $dbForProject, $database) {
-            $document->removeAttribute('$collection');
-            $document->setAttribute('$databaseId', $database->getId());
-            $document->setAttribute('$collectionId', $table->getId());
-
-            $relationships = \array_filter(
-                $table->getAttribute('attributes', []),
-                fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP
-            );
-
-            foreach ($relationships as $relationship) {
-                $related = $document->getAttribute($relationship->getAttribute('key'));
-
-                if (empty($related)) {
-                    continue;
-                }
-                if (!\is_array($related)) {
-                    $related = [$related];
-                }
-
-                $relatedCollectionId = $relationship->getAttribute('relatedCollection');
-                $relatedCollection = Authorization::skip(
-                    fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $relatedCollectionId)
-                );
-
-                foreach ($related as $relation) {
-                    if ($relation instanceof Document) {
-                        $processDocument($relatedCollection, $relation);
-                    }
-                }
-            }
-        };
-
+        $collectionsCache = [];
        foreach ($documents as $document) {
-            $processDocument($collection, $document);
+            $this->processDocument(
+                database: $database,
+                collection: $collection,
+                document: $document,
+                dbForProject: $dbForProject,
+                collectionsCache: $collectionsCache,
+            );
        }

        $queueForStatsUsage
--- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php
+++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php
@ -12,7 +12,6 @@ use Appwrite\SDK\Method;
 use Appwrite\SDK\Response as SDKResponse;
 use Appwrite\Utopia\Response as UtopiaResponse;
 use Utopia\Database\Database;
-use Utopia\Database\Document;
 use Utopia\Database\Exception\Conflict as ConflictException;
 use Utopia\Database\Exception\Restricted as RestrictedException;
 use Utopia\Database\Validator\Authorization;
@ -115,40 +114,14 @@ class Delete extends Action
            throw new Exception($this->getRestrictedException());
        }

-        // Add $collection and $databaseId for all documents
-        $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) {
-            $document->setAttribute('$databaseId', $database->getId());
-            $document->setAttribute('$collectionId', $collection->getId());
-
-            $relationships = \array_filter(
-                $collection->getAttribute('attributes', []),
-                fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP
-            );
-
-            foreach ($relationships as $relationship) {
-                $related = $document->getAttribute($relationship->getAttribute('key'));
-
-                if (empty($related)) {
-                    continue;
-                }
-                if (!\is_array($related)) {
-                    $related = [$related];
-                }
-
-                $relatedCollectionId = $relationship->getAttribute('relatedCollection');
-                $relatedCollection = Authorization::skip(
-                    fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $relatedCollectionId)
-                );
-
-                foreach ($related as $relation) {
-                    if ($relation instanceof Document) {
-                        $processDocument($relatedCollection, $relation);
-                    }
-                }
-            }
-        };
-
-        $processDocument($collection, $document);
+        $collectionsCache = [];
+        $this->processDocument(
+            database: $database,
+            collection: $collection,
+            document: $document,
+            dbForProject: $dbForProject,
+            collectionsCache: $collectionsCache,
+        );

        $queueForStatsUsage
            ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1)
--- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php
+++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php
@ -11,7 +11,6 @@ use Appwrite\SDK\Method;
 use Appwrite\SDK\Response as SDKResponse;
 use Appwrite\Utopia\Response as UtopiaResponse;
 use Utopia\Database\Database;
-use Utopia\Database\Document;
 use Utopia\Database\Exception\Query as QueryException;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
@ -101,52 +100,15 @@ class Get extends Action
        }

        $operations = 0;
-
-        // Add $collectionId and $databaseId for all rows
-        $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) {
-            if ($document->isEmpty()) {
-                return;
-            }
-
-            $operations++;
-
-            $document->setAttribute('$databaseId', $database->getId());
-            $document->setAttribute('$collectionId', $collection->getId());
-
-            $relationships = \array_filter(
-                $collection->getAttribute('attributes', []),
-                fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP
-            );
-
-            foreach ($relationships as $relationship) {
-                $related = $document->getAttribute($relationship->getAttribute('key'));
-
-                if (empty($related)) {
-                    if (\in_array(\gettype($related), ['array', 'object'])) {
-                        $operations++;
-                    }
-
-                    continue;
-                }
-
-                if (!\is_array($related)) {
-                    $related = [$related];
-                }
-
-                $relatedCollectionId = $relationship->getAttribute('relatedCollection');
-                $relatedCollection = Authorization::skip(
-                    fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $relatedCollectionId)
-                );
-
-                foreach ($related as $relation) {
-                    if ($relation instanceof Document) {
-                        $processDocument($relatedCollection, $relation);
-                    }
-                }
-            }
-        };
-
-        $processDocument($collection, $document);
+        $collectionsCache = [];
+        $this->processDocument(
+            database: $database,
+            collection: $collection,
+            document: $document,
+            dbForProject: $dbForProject,
+            collectionsCache: $collectionsCache,
+            operations: $operations
+        );

        $queueForStatsUsage
            ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1))
--- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php
+++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php
@ -247,40 +247,14 @@ class Update extends Action
            throw new Exception($this->getInvalidStructureException(), $e->getMessage());
        }

-        // Add $collectionId and $databaseId for all documents
-        $processDocument = function (Document $table, Document $row) use (&$processDocument, $dbForProject, $database) {
-            $row->setAttribute('$databaseId', $database->getId());
-            $row->setAttribute('$collectionId', $table->getId());
-
-            $relationships = \array_filter(
-                $table->getAttribute('attributes', []),
-                fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP
-            );
-
-            foreach ($relationships as $relationship) {
-                $related = $row->getAttribute($relationship->getAttribute('key'));
-
-                if (empty($related)) {
-                    continue;
-                }
-                if (!\is_array($related)) {
-                    $related = [$related];
-                }
-
-                $relatedCollectionId = $relationship->getAttribute('relatedCollection');
-                $relatedCollection = Authorization::skip(
-                    fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $relatedCollectionId)
-                );
-
-                foreach ($related as $relation) {
-                    if ($relation instanceof Document) {
-                        $processDocument($relatedCollection, $relation);
-                    }
-                }
-            }
-        };
-
-        $processDocument($collection, $document);
+        $collectionsCache = [];
+        $this->processDocument(
+            database: $database,
+            collection: $collection,
+            document: $document,
+            dbForProject: $dbForProject,
+            collectionsCache: $collectionsCache,
+        );

        $response->dynamic($document, $this->getResponseModel());

--- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php
+++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php
@ -235,41 +235,15 @@ class Upsert extends Action
            throw new Exception($this->getInvalidStructureException(), $e->getMessage());
        }

+        $collectionsCache = [];
        $document = $upserted[0];
-        // Add $collectionId and $databaseId for all documents
-        $processDocument = function (Document $table, Document $document) use (&$processDocument, $dbForProject, $database) {
-            $document->setAttribute('$databaseId', $database->getId());
-            $document->setAttribute('$collectionId', $table->getId());
-
-            $relationships = \array_filter(
-                $table->getAttribute('attributes', []),
-                fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP
-            );
-
-            foreach ($relationships as $relationship) {
-                $related = $document->getAttribute($relationship->getAttribute('key'));
-
-                if (empty($related)) {
-                    continue;
-                }
-                if (!\is_array($related)) {
-                    $related = [$related];
-                }
-
-                $relatedCollectionId = $relationship->getAttribute('relatedCollection');
-                $relatedCollection = Authorization::skip(
-                    fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $relatedCollectionId)
-                );
-
-                foreach ($related as $relation) {
-                    if ($relation instanceof Document) {
-                        $processDocument($relatedCollection, $relation);
-                    }
-                }
-            }
-        };
-
-        $processDocument($collection, $document);
+        $this->processDocument(
+            database: $database,
+            collection: $collection,
+            document: $document,
+            dbForProject: $dbForProject,
+            collectionsCache: $collectionsCache,
+        );

        $relationships = \array_map(
            fn ($document) => $document->getAttribute('key'),
--- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php
+++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php
@ -130,65 +130,16 @@ class XList extends Action
        }

        $operations = 0;
-
-        // Add $collectionId and $databaseId for all documents
-        $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool {
-            if ($document->isEmpty()) {
-                return false;
-            }
-
-            $operations++;
-
-            $document->removeAttribute('$collection');
-            $document->setAttribute('$databaseId', $database->getId());
-            $document->setAttribute('$collectionId', $collection->getId());
-
-            $relationships = \array_filter(
-                $collection->getAttribute('attributes', []),
-                fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP
+        $collectionsCache = [];
+        foreach ($documents as $document) {
+            $this->processDocument(
+                database: $database,
+                collection: $collection,
+                document: $document,
+                dbForProject: $dbForProject,
+                collectionsCache: $collectionsCache,
+                operations: $operations,
            );
-
-            foreach ($relationships as $relationship) {
-                $related = $document->getAttribute($relationship->getAttribute('key'));
-
-                if (empty($related)) {
-                    if (\in_array(\gettype($related), ['array', 'object'])) {
-                        $operations++;
-                    }
-
-                    continue;
-                }
-
-                if (!\is_array($related)) {
-                    $relations = [$related];
-                } else {
-                    $relations = $related;
-                }
-
-                $relatedCollectionId = $relationship->getAttribute('relatedCollection');
-                // todo: Use local cache for this getDocument
-                $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $relatedCollectionId));
-
-                foreach ($relations as $index => $doc) {
-                    if ($doc instanceof Document) {
-                        if (!$processDocument($relatedCollection, $doc)) {
-                            unset($relations[$index]);
-                        }
-                    }
-                }
-
-                if (\is_array($related)) {
-                    $document->setAttribute($relationship->getAttribute('key'), \array_values($relations));
-                } elseif (empty($relations)) {
-                    $document->setAttribute($relationship->getAttribute('key'), null);
-                }
-            }
-
-            return true;
-        });
-
-        foreach ($documents as $row) {
-            $processDocument($collection, $row);
        }

        $queueForStatsUsage