From 92a04535ac41b5398821efeeb03d4fcd921787a7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 14:35:17 -0400 Subject: [PATCH 1/8] Add database worker job logging --- src/Appwrite/Platform/Workers/Databases.php | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 37f1c3212a..fb0a4497aa 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Realtime; use Exception; +use Swoole\Timer; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; @@ -39,6 +40,10 @@ class Databases extends Action ->inject('queueForRealtime') ->inject('log') ->callback($this->action(...)); + + Timer::tick(10_000, function () { + Console::info('Databases worker heartbeat'); + }); } /** @@ -64,6 +69,14 @@ class Databases extends Action $document = new Document($payload['document'] ?? []); $database = new Document($payload['database'] ?? []); + Console::info("Processing database operation: \n" . \json_encode([ + 'type' => $type, + 'projectId' => $project->getId(), + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId(), + 'documentId' => $document->getId(), + ], JSON_PRETTY_PRINT)); + $log->addTag('projectId', $project->getId()); $log->addTag('type', $type); @@ -82,6 +95,14 @@ class Databases extends Action DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), default => throw new Exception('No database operation for type: ' . \strval($type)), }; + + Console::info("Finished processing database operation: \n" . \json_encode([ + 'type' => $type, + 'projectId' => $project->getId(), + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId(), + 'documentId' => $document->getId(), + ], JSON_PRETTY_PRINT)); } /** From d7d7fa3096140819d60122394cb4795528ec780c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 14:47:23 -0400 Subject: [PATCH 2/8] Add trigger logging --- app/controllers/shared/api.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index f99ebbce07..772b83fff5 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -22,6 +22,7 @@ use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; +use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -797,6 +798,10 @@ App::shutdown() } if (!empty($queueForDatabase->getType())) { + Console::info("Triggering database event: \n" . \json_encode([ + 'type' => $queueForDatabase->getType(), + 'project' => $project->getId(), + ])); $queueForDatabase->trigger(); } From e37cea514658b1107b84c53dc33e0f3921baf19f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 14:50:29 -0400 Subject: [PATCH 3/8] Add more trigger logging --- app/controllers/shared/api.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 772b83fff5..fd2536805d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -799,8 +799,10 @@ App::shutdown() if (!empty($queueForDatabase->getType())) { Console::info("Triggering database event: \n" . \json_encode([ - 'type' => $queueForDatabase->getType(), - 'project' => $project->getId(), + 'projectId' => $project->getId(), + 'databaseId' => $queueForDatabase->getDatabase()->getId(), + 'collectionId' => $queueForDatabase->getCollection()->getId(), + 'documentId' => $queueForDatabase->getDocument()->getId(), ])); $queueForDatabase->trigger(); } From 255be3babf920921a58d42b1a3931cb2a22cb978 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 14:52:14 -0400 Subject: [PATCH 4/8] Ensure heartbeat is cleared on destruct --- src/Appwrite/Event/Database.php | 10 ++++++++++ src/Appwrite/Platform/Workers/Databases.php | 13 ++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index d2f70dddf2..838870345a 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -54,6 +54,16 @@ class Database extends Event return $this; } + /** + * Returns set database for this event. + * + * @return null|Document + */ + public function getDatabase(): ?Document + { + return $this->database; + } + /** * Set the collection for this database event. * diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index fb0a4497aa..0ba55ad97f 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -21,6 +21,8 @@ use Utopia\Queue\Message; class Databases extends Action { + private ?int $heartbeatId = null; + public static function getName(): string { return 'databases'; @@ -41,11 +43,19 @@ class Databases extends Action ->inject('log') ->callback($this->action(...)); - Timer::tick(10_000, function () { + $this->heartbeatId = Timer::tick(10_000, function () { Console::info('Databases worker heartbeat'); }); } + public function __destruct() + { + if ($this->heartbeatId) { + Timer::clear($this->heartbeatId); + $this->heartbeatId = null; + } + } + /** * @param Message $message * @param Document $project @@ -137,6 +147,7 @@ class Databases extends Action $projectId = $project->getId(); $event = "databases.[databaseId].collections.[collectionId].attributes.[attributeId].update"; + /** * TODO @christyjacob4 verify if this is still the case * Fetch attribute from the database, since with Resque float values are loosing informations. From d52d8f7697879db4d87877d003c9e7553ef32da3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 15:47:57 -0400 Subject: [PATCH 5/8] Fix API dump --- app/controllers/shared/api.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index fd2536805d..735487339f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -800,9 +800,9 @@ App::shutdown() if (!empty($queueForDatabase->getType())) { Console::info("Triggering database event: \n" . \json_encode([ 'projectId' => $project->getId(), - 'databaseId' => $queueForDatabase->getDatabase()->getId(), - 'collectionId' => $queueForDatabase->getCollection()->getId(), - 'documentId' => $queueForDatabase->getDocument()->getId(), + 'databaseId' => $queueForDatabase->getDatabase()?->getId(), + 'collectionId' => $queueForDatabase->getCollection()?->getId(), + 'documentId' => $queueForDatabase->getDocument()?->getId(), ])); $queueForDatabase->trigger(); } From f0ea206f2957a0b23650fddbef4762d96788117d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 15:56:07 -0400 Subject: [PATCH 6/8] Fix bench --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 562f2223f3..f1051497a8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -65,7 +65,7 @@ jobs: sudo apt update sudo apt install oha - name: Benchmark PR - run: 'oha -z 180s http://localhost/v1/health/version --output_format json > benchmark.json' + run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' - name: Cleaning run: docker compose down -v - name: Installing latest version @@ -78,7 +78,7 @@ jobs: docker compose up -d sleep 10 - name: Benchmark Latest - run: oha -z 180s http://localhost/v1/health/version --output_format json > benchmark-latest.json + run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json - name: Prepare comment run: | echo '## :sparkles: Benchmark results' > benchmark.txt From 620266d9d607641e4db1dcb63f513b30e54d4f26 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 16:40:02 -0400 Subject: [PATCH 7/8] Remove heartbeat --- src/Appwrite/Platform/Workers/Databases.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 0ba55ad97f..bf71ba3946 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -21,8 +21,6 @@ use Utopia\Queue\Message; class Databases extends Action { - private ?int $heartbeatId = null; - public static function getName(): string { return 'databases'; @@ -42,18 +40,6 @@ class Databases extends Action ->inject('queueForRealtime') ->inject('log') ->callback($this->action(...)); - - $this->heartbeatId = Timer::tick(10_000, function () { - Console::info('Databases worker heartbeat'); - }); - } - - public function __destruct() - { - if ($this->heartbeatId) { - Timer::clear($this->heartbeatId); - $this->heartbeatId = null; - } } /** From 7508a22ddfe45123282d8d3fc5a7d806b3b4f5ad Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Jun 2025 16:49:12 -0400 Subject: [PATCH 8/8] Lint --- composer.lock | 74 +++++++++++---------- src/Appwrite/Platform/Workers/Databases.php | 1 - 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/composer.lock b/composer.lock index af7a609930..fd6a322076 100644 --- a/composer.lock +++ b/composer.lock @@ -1113,16 +1113,16 @@ }, { "name": "open-telemetry/api", - "version": "1.3.0", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "4e3bb38e069876fb73c2ce85c89583bf2b28cd86" + "reference": "b3a9286f9c1c8247c83493c5b1fa475cd0cec7f7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/4e3bb38e069876fb73c2ce85c89583bf2b28cd86", - "reference": "4e3bb38e069876fb73c2ce85c89583bf2b28cd86", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/b3a9286f9c1c8247c83493c5b1fa475cd0cec7f7", + "reference": "b3a9286f9c1c8247c83493c5b1fa475cd0cec7f7", "shasum": "" }, "require": { @@ -1142,7 +1142,7 @@ ] }, "branch-alias": { - "dev-main": "1.1.x-dev" + "dev-main": "1.4.x-dev" } }, "autoload": { @@ -1179,7 +1179,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-07T12:32:21+00:00" + "time": "2025-06-19T23:36:51+00:00" }, { "name": "open-telemetry/context", @@ -1242,16 +1242,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.3.1", + "version": "1.3.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "8b3ca1f86d01429c73b407bf1a2075d9c187001e" + "reference": "196f3a1dbce3b2c0f8110d164232c11ac00ddbb2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/8b3ca1f86d01429c73b407bf1a2075d9c187001e", - "reference": "8b3ca1f86d01429c73b407bf1a2075d9c187001e", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/196f3a1dbce3b2c0f8110d164232c11ac00ddbb2", + "reference": "196f3a1dbce3b2c0f8110d164232c11ac00ddbb2", "shasum": "" }, "require": { @@ -1302,7 +1302,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-21T12:02:20+00:00" + "time": "2025-06-16T00:24:51+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -1369,22 +1369,22 @@ }, { "name": "open-telemetry/sdk", - "version": "1.5.0", + "version": "1.6.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "cd0d7367599717fc29e04eb8838ec061e6c2c657" + "reference": "1c0371794e4c0700afd4a9d4d8511cb5e3f78e6a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/cd0d7367599717fc29e04eb8838ec061e6c2c657", - "reference": "cd0d7367599717fc29e04eb8838ec061e6c2c657", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/1c0371794e4c0700afd4a9d4d8511cb5e3f78e6a", + "reference": "1c0371794e4c0700afd4a9d4d8511cb5e3f78e6a", "shasum": "" }, "require": { "ext-json": "*", "nyholm/psr7-server": "^1.1", - "open-telemetry/api": "~1.0 || ~1.1", + "open-telemetry/api": "~1.4.0", "open-telemetry/context": "^1.0", "open-telemetry/sem-conv": "^1.0", "php": "^8.1", @@ -1407,6 +1407,10 @@ "type": "library", "extra": { "spi": { + "OpenTelemetry\\API\\Configuration\\ConfigEnv\\EnvComponentLoader": [ + "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderHttpConfig", + "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderPeerConfig" + ], "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] @@ -1455,7 +1459,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-22T02:33:34+00:00" + "time": "2025-06-19T23:36:51+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1874,16 +1878,16 @@ }, { "name": "phpseclib/phpseclib", - "version": "3.0.44", + "version": "3.0.45", "source": { "type": "git", "url": "https://github.com/phpseclib/phpseclib.git", - "reference": "1d0b5e7e1434678411787c5a0535e68907cf82d9" + "reference": "bd81b90d5963c6b9d87de50357585375223f4dd8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/1d0b5e7e1434678411787c5a0535e68907cf82d9", - "reference": "1d0b5e7e1434678411787c5a0535e68907cf82d9", + "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/bd81b90d5963c6b9d87de50357585375223f4dd8", + "reference": "bd81b90d5963c6b9d87de50357585375223f4dd8", "shasum": "" }, "require": { @@ -1964,7 +1968,7 @@ ], "support": { "issues": "https://github.com/phpseclib/phpseclib/issues", - "source": "https://github.com/phpseclib/phpseclib/tree/3.0.44" + "source": "https://github.com/phpseclib/phpseclib/tree/3.0.45" }, "funding": [ { @@ -1980,7 +1984,7 @@ "type": "tidelift" } ], - "time": "2025-06-15T09:59:26+00:00" + "time": "2025-06-22T22:54:43+00:00" }, { "name": "psr/container", @@ -4807,16 +4811,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.41.6", + "version": "0.41.8", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "bfcebb968c527e17fdf18d40b8986c83d9c18c93" + "reference": "93ffb24b25b376ca4423e3a5caf6f916673af4b2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/bfcebb968c527e17fdf18d40b8986c83d9c18c93", - "reference": "bfcebb968c527e17fdf18d40b8986c83d9c18c93", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/93ffb24b25b376ca4423e3a5caf6f916673af4b2", + "reference": "93ffb24b25b376ca4423e3a5caf6f916673af4b2", "shasum": "" }, "require": { @@ -4852,9 +4856,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.41.6" + "source": "https://github.com/appwrite/sdk-generator/tree/0.41.8" }, - "time": "2025-06-12T03:27:26+00:00" + "time": "2025-06-18T13:20:45+00:00" }, { "name": "doctrine/annotations", @@ -5147,16 +5151,16 @@ }, { "name": "matthiasmullie/minify", - "version": "1.3.74", + "version": "1.3.75", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "a2593286a4135d03c6a6a9e9aeded5d41e931ce4" + "reference": "76ba4a5f555fd7bf4aa408af608e991569076671" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/a2593286a4135d03c6a6a9e9aeded5d41e931ce4", - "reference": "a2593286a4135d03c6a6a9e9aeded5d41e931ce4", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/76ba4a5f555fd7bf4aa408af608e991569076671", + "reference": "76ba4a5f555fd7bf4aa408af608e991569076671", "shasum": "" }, "require": { @@ -5205,7 +5209,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.74" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.75" }, "funding": [ { @@ -5213,7 +5217,7 @@ "type": "github" } ], - "time": "2025-06-12T08:06:04+00:00" + "time": "2025-06-25T09:56:19+00:00" }, { "name": "matthiasmullie/path-converter", diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index bf71ba3946..896ec6d32d 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -4,7 +4,6 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Realtime; use Exception; -use Swoole\Timer; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document;