From eb2c3cd73fb15a8af98ad6978e1e3350da94c732 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:55:38 +0000 Subject: [PATCH 01/48] chore: platform bump --- app/config/platforms.php | 8 ++--- composer.lock | 70 ++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/app/config/platforms.php b/app/config/platforms.php index e54fb0a073..1470bd1364 100644 --- a/app/config/platforms.php +++ b/app/config/platforms.php @@ -11,7 +11,7 @@ return [ [ 'key' => 'web', 'name' => 'Web', - 'version' => '16.0.2', + 'version' => '16.1.0', 'url' => 'https://github.com/appwrite/sdk-for-web', 'package' => 'https://www.npmjs.com/package/appwrite', 'enabled' => true, @@ -59,7 +59,7 @@ return [ [ 'key' => 'flutter', 'name' => 'Flutter', - 'version' => '13.0.0', + 'version' => '13.1.0', 'url' => 'https://github.com/appwrite/sdk-for-flutter', 'package' => 'https://pub.dev/packages/appwrite', 'enabled' => true, @@ -77,7 +77,7 @@ return [ [ 'key' => 'apple', 'name' => 'Apple', - 'version' => '7.0.0', + 'version' => '7.1.0', 'url' => 'https://github.com/appwrite/sdk-for-apple', 'package' => 'https://github.com/appwrite/sdk-for-apple', 'enabled' => true, @@ -112,7 +112,7 @@ return [ [ 'key' => 'android', 'name' => 'Android', - 'version' => '6.0.0', + 'version' => '6.1.0', 'url' => 'https://github.com/appwrite/sdk-for-android', 'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-android', 'enabled' => true, diff --git a/composer.lock b/composer.lock index ee947d27f0..986a534435 100644 --- a/composer.lock +++ b/composer.lock @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.1.1", + "version": "1.1.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6" + "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/542064815d38a6df55af7957cd6f1d7d967c99c6", - "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", + "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", "shasum": "" }, "require": { @@ -1260,13 +1260,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.1.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.1.x-dev" } }, "autoload": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-10-15T22:42:37+00:00" + "time": "2024-11-16T04:32:30+00:00" }, { "name": "open-telemetry/context", @@ -2453,23 +2453,23 @@ }, { "name": "symfony/http-client", - "version": "v7.2.0", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "955e43336aff03df1e8a8e17daefabb0127a313b" + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/955e43336aff03df1e8a8e17daefabb0127a313b", - "reference": "955e43336aff03df1e8a8e17daefabb0127a313b", + "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", "shasum": "" }, "require": { "php": ">=8.2", "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", - "symfony/http-client-contracts": "~3.4.3|^3.5.1", + "symfony/http-client-contracts": "~3.4.4|^3.5.2", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.0" + "source": "https://github.com/symfony/http-client/tree/v7.2.1" }, "funding": [ { @@ -2544,20 +2544,20 @@ "type": "tidelift" } ], - "time": "2024-11-29T08:22:02+00:00" + "time": "2024-12-07T08:50:44+00:00" }, { "name": "symfony/http-client-contracts", - "version": "v3.5.1", + "version": "v3.5.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client-contracts.git", - "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9" + "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/c2f3ad828596624ca39ea40f83617ef51ca8bbf9", - "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/ee8d807ab20fcb51267fdace50fbe3494c31e645", + "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645", "shasum": "" }, "require": { @@ -2606,7 +2606,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.1" + "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.2" }, "funding": [ { @@ -2622,7 +2622,7 @@ "type": "tidelift" } ], - "time": "2024-11-25T12:02:18+00:00" + "time": "2024-12-07T08:49:48+00:00" }, { "name": "symfony/polyfill-mbstring", @@ -2650,8 +2650,8 @@ "type": "library", "extra": { "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" } }, "autoload": { @@ -4807,16 +4807,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.25", + "version": "0.39.26", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1" + "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/5b5323636a8d75a1c4faaae9728098dd6a6a47d1", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/39768deacb4913f93548c46fa0149c3fadc62d0c", + "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c", "shasum": "" }, "require": { @@ -4852,9 +4852,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.25" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.26" }, - "time": "2024-11-08T10:16:34+00:00" + "time": "2024-12-12T10:51:34+00:00" }, { "name": "doctrine/annotations", @@ -7576,16 +7576,16 @@ }, { "name": "symfony/console", - "version": "v7.2.0", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf" + "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", - "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", + "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", "shasum": "" }, "require": { @@ -7649,7 +7649,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.0" + "source": "https://github.com/symfony/console/tree/v7.2.1" }, "funding": [ { @@ -7665,7 +7665,7 @@ "type": "tidelift" } ], - "time": "2024-11-06T14:24:19+00:00" + "time": "2024-12-11T03:49:26+00:00" }, { "name": "symfony/filesystem", From 6372cf59ccbbe4b20530397c3ef25ce881fecb5b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:54:50 +0000 Subject: [PATCH 02/48] chore: composer --- composer.json | 2 +- composer.lock | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/composer.json b/composer.json index 512e203a5e..fdbf2b4514 100644 --- a/composer.json +++ b/composer.json @@ -84,7 +84,7 @@ }, "require-dev": { "ext-fileinfo": "*", - "appwrite/sdk-generator": "0.39.*", + "appwrite/sdk-generator": "0.39.27", "phpunit/phpunit": "9.5.20", "swoole/ide-helper": "5.1.2", "textalk/websocket": "1.5.7", diff --git a/composer.lock b/composer.lock index 986a534435..ac77c575ec 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fae350df93342992edd8f639948e1570", + "content-hash": "35d6e3f5c403194ad7969d48dc4cbdf5", "packages": [ { "name": "adhocore/jwt", @@ -1530,13 +1530,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.0.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.0.x-dev" } }, "autoload": { @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.1", + "version": "0.43.2", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa" + "reference": "374536b86d8d39066960a7da161d444a099bbc56" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/e404c21e8dcf6a310bc83cf1d74e716b105598fa", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/374536b86d8d39066960a7da161d444a099bbc56", + "reference": "374536b86d8d39066960a7da161d444a099bbc56", "shasum": "" }, "require": { @@ -3153,7 +3153,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.53.*" + "utopia-php/database": "0.53.200" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.1" + "source": "https://github.com/utopia-php/abuse/tree/0.43.2" }, - "time": "2024-10-23T04:29:12+00:00" + "time": "2024-12-12T19:43:24+00:00" }, { "name": "utopia-php/analytics", @@ -4807,16 +4807,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.26", + "version": "0.39.27", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c" + "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/39768deacb4913f93548c46fa0149c3fadc62d0c", - "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/27d8ecde30e40cbfe1124cc0430c406d3e144849", + "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849", "shasum": "" }, "require": { @@ -4852,9 +4852,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.26" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.27" }, - "time": "2024-12-12T10:51:34+00:00" + "time": "2024-12-16T11:32:02+00:00" }, { "name": "doctrine/annotations", From 47785d9c4de9a53f534192bd7e6563ec71bc9f3b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:58:25 +0000 Subject: [PATCH 03/48] docs: update --- .../1.6.x/client-graphql/examples/account/create-push-target.md | 1 + docs/examples/1.6.x/client-graphql/examples/account/create.md | 1 + docs/examples/1.6.x/client-graphql/examples/account/get.md | 1 + .../1.6.x/client-graphql/examples/account/update-email.md | 1 + .../1.6.x/client-graphql/examples/account/update-m-f-a.md | 1 + .../client-graphql/examples/account/update-mfa-authenticator.md | 1 + .../1.6.x/client-graphql/examples/account/update-name.md | 1 + .../1.6.x/client-graphql/examples/account/update-password.md | 1 + .../1.6.x/client-graphql/examples/account/update-phone.md | 1 + .../1.6.x/client-graphql/examples/account/update-prefs.md | 1 + .../1.6.x/client-graphql/examples/account/update-push-target.md | 1 + .../1.6.x/client-graphql/examples/account/update-status.md | 1 + .../1.6.x/client-graphql/examples/messaging/create-subscriber.md | 1 + 13 files changed, 13 insertions(+) diff --git a/docs/examples/1.6.x/client-graphql/examples/account/create-push-target.md b/docs/examples/1.6.x/client-graphql/examples/account/create-push-target.md index 8a0fad387c..63802a782e 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/create-push-target.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/create-push-target.md @@ -12,5 +12,6 @@ mutation { providerId providerType identifier + expired } } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/create.md b/docs/examples/1.6.x/client-graphql/examples/account/create.md index 3f8e3c3cf7..0d39394a3d 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/create.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/create.md @@ -33,6 +33,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/get.md b/docs/examples/1.6.x/client-graphql/examples/account/get.md index e4db8f0e41..f4f07c187f 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/get.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/get.md @@ -28,6 +28,7 @@ query { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-email.md b/docs/examples/1.6.x/client-graphql/examples/account/update-email.md index b207bad4bb..c879e24a43 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-email.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-email.md @@ -31,6 +31,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-m-f-a.md b/docs/examples/1.6.x/client-graphql/examples/account/update-m-f-a.md index d2cd3d6ae5..787c2e0860 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-m-f-a.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-m-f-a.md @@ -30,6 +30,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-authenticator.md b/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-authenticator.md index c74062c7d4..9cfe9150be 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-authenticator.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-authenticator.md @@ -31,6 +31,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-name.md b/docs/examples/1.6.x/client-graphql/examples/account/update-name.md index 850b5760a0..8ba2c99d9c 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-name.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-name.md @@ -30,6 +30,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-password.md b/docs/examples/1.6.x/client-graphql/examples/account/update-password.md index 5904da0842..f3619a10d2 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-password.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-password.md @@ -31,6 +31,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-phone.md b/docs/examples/1.6.x/client-graphql/examples/account/update-phone.md index 408a203300..adecb71168 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-phone.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-phone.md @@ -31,6 +31,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-prefs.md b/docs/examples/1.6.x/client-graphql/examples/account/update-prefs.md index 40db7b43db..57280247e4 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-prefs.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-prefs.md @@ -30,6 +30,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-push-target.md b/docs/examples/1.6.x/client-graphql/examples/account/update-push-target.md index 059702dc97..3c402cdf12 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-push-target.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-push-target.md @@ -11,5 +11,6 @@ mutation { providerId providerType identifier + expired } } diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-status.md b/docs/examples/1.6.x/client-graphql/examples/account/update-status.md index aca8c098e7..c17f556842 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-status.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-status.md @@ -28,6 +28,7 @@ mutation { providerId providerType identifier + expired } accessedAt } diff --git a/docs/examples/1.6.x/client-graphql/examples/messaging/create-subscriber.md b/docs/examples/1.6.x/client-graphql/examples/messaging/create-subscriber.md index b2712ebb48..bab53612b7 100644 --- a/docs/examples/1.6.x/client-graphql/examples/messaging/create-subscriber.md +++ b/docs/examples/1.6.x/client-graphql/examples/messaging/create-subscriber.md @@ -17,6 +17,7 @@ mutation { providerId providerType identifier + expired } userId userName From 53860d2b96b21d4f98e67d4d4c6b7db182e3310f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 10:07:53 +0000 Subject: [PATCH 04/48] chore: remove abuse cleanup --- src/Appwrite/Platform/Workers/Deletes.php | 31 +++-------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index f29b79e5db..9aaf19f412 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -7,7 +7,6 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; -use Utopia\Abuse\Abuse; use Utopia\Audit\Audit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -52,13 +51,12 @@ class Deletes extends Action ->inject('deviceForBuilds') ->inject('deviceForCache') ->inject('certificates') - ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, int $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +64,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, int $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -125,9 +123,6 @@ class Deletes extends Action $this->deleteAuditLogs($project, $getProjectDB, $auditRetention); } break; - case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($project, $timelimit, $abuseRetention); - break; case DELETE_TYPE_REALTIME: $this->deleteRealtimeUsage($dbForPlatform, $datetime); break; @@ -700,26 +695,6 @@ class Deletes extends Action ], $dbForPlatform); } - /** - * @param Database $dbForPlatform - * @param callable $getProjectDB - * @param int $abuseRetention - * @return void - * @throws Exception - */ - private function deleteAbuseLogs(Document $project, callable $timelimit, int $abuseRetention): void - { - $projectId = $project->getId(); - $timeLimit = $timelimit("", 0, 1); - $abuse = new Abuse($timeLimit); - - try { - $abuse->cleanup($abuseRetention); - } catch (DatabaseException $e) { - Console::error('Failed to delete abuse logs for project ' . $projectId . ': ' . $e->getMessage()); - } - } - /** * @param Database $dbForPlatform * @param callable $getProjectDB From 769772a0a840911db1995464897567b4b253650e Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 20:30:03 +0000 Subject: [PATCH 05/48] fix: remove abuse delete trigger --- src/Appwrite/Platform/Tasks/Maintenance.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 55f4f25843..c789cbdaac 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -66,7 +66,6 @@ class Maintenance extends Action { $this->notifyDeleteTargets($queueForDeletes); $this->notifyDeleteExecutionLogs($queueForDeletes); - $this->notifyDeleteAbuseLogs($queueForDeletes); $this->notifyDeleteAuditLogs($queueForDeletes); $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes); $this->notifyDeleteExpiredSessions($queueForDeletes); @@ -106,13 +105,6 @@ class Maintenance extends Action ->trigger(); } - private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_ABUSE) - ->trigger(); - } - private function notifyDeleteAuditLogs(Delete $queueForDeletes): void { $queueForDeletes From 4691407a7abe31270d38cebdf07a24faef3fdc4c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 20:41:30 +0000 Subject: [PATCH 06/48] chore: bump utopia-php/abuse to 0.46.0 --- composer.json | 2 +- composer.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/composer.json b/composer.json index 2879993d3f..e49948977b 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.45.*", + "utopia-php/abuse": "0.46.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 5bce56a022..083456a0d4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "db3b63ba1be2561ee4a112e5b9c6bd53", + "content-hash": "786a6c2f9aa130c673f8a39a18a28e3c", "packages": [ { "name": "adhocore/jwt", @@ -2403,12 +2403,12 @@ }, "type": "library", "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, "branch-alias": { "dev-main": "3.5-dev" - }, - "thanks": { - "name": "symfony/contracts", - "url": "https://github.com/symfony/contracts" } }, "autoload": { @@ -2884,12 +2884,12 @@ }, "type": "library", "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, "branch-alias": { "dev-main": "3.5-dev" - }, - "thanks": { - "name": "symfony/contracts", - "url": "https://github.com/symfony/contracts" } }, "autoload": { @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.45.0", + "version": "0.46.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "54a58355df1eb26c9be7834738debe6754e5581d" + "reference": "bef426a9a28b3e71b08216bcbe310455f588c11b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/54a58355df1eb26c9be7834738debe6754e5581d", - "reference": "54a58355df1eb26c9be7834738debe6754e5581d", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/bef426a9a28b3e71b08216bcbe310455f588c11b", + "reference": "bef426a9a28b3e71b08216bcbe310455f588c11b", "shasum": "" }, "require": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.45.0" + "source": "https://github.com/utopia-php/abuse/tree/0.46.0" }, - "time": "2024-12-20T16:18:52+00:00" + "time": "2024-12-27T17:46:08+00:00" }, { "name": "utopia-php/analytics", From 58fd0d35bd13d034c79efc2d2ee6b4210ac75955 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:47:52 +0200 Subject: [PATCH 07/48] database crud usage addition --- src/Appwrite/Event/Usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 89e900d2ab..d2e302f82b 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -69,7 +69,7 @@ class Usage extends Event 'reduce' => $this->reduce, 'metrics' => $this->metrics, ]); - + var_dump($this->metrics); $this->metrics = []; return $result; From c23af8168becf391dcc8403d10d727fe553fdeab Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:48:05 +0200 Subject: [PATCH 08/48] database crud usage addition --- app/controllers/api/databases.php | 158 +++++++++++++++++++----------- app/init.php | 2 + 2 files changed, 104 insertions(+), 56 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index aad072c50a..80bc72c4b7 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2885,7 +2885,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is not allowed for creating new documents, try update instead'); } - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('databases', $databaseId); + }); + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -2895,6 +2899,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -2981,8 +2986,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($relations as &$relation) { @@ -2995,8 +3002,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relation = new Document($relation); } if ($relation instanceof Document) { - $current = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()) + $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); + } ); if ($current->isEmpty()) { @@ -3028,6 +3037,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3037,7 +3047,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3057,8 +3067,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($related as $relation) { @@ -3116,8 +3128,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3126,6 +3141,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3155,7 +3171,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); - $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($cursorDocument->isEmpty()) { throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Document '{$documentId}' for the 'cursor' value not found."); @@ -3165,10 +3184,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { if ($document->isEmpty()) { return false; } @@ -3195,7 +3217,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); + $relatedCollection = Authorization::skip(function() use ($dbForProject, $queueForUsage, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($relations as $index => $doc) { if ($doc instanceof Document) { @@ -3274,8 +3299,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3284,7 +3312,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3293,6 +3324,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3304,7 +3336,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { if ($document->isEmpty()) { return; } @@ -3328,8 +3360,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($related as $relation) { @@ -3481,7 +3515,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('dbForProject') ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Usage $queueForUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3489,7 +3524,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DOCUMENT_MISSING_PAYLOAD); } - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('databases', $databaseId); + }); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3498,7 +3536,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3506,7 +3547,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ - $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); @@ -3548,7 +3592,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, $queueForUsage) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3570,9 +3614,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($relations as &$relation) { // If the relation is an array it can be either update or create a child document. @@ -3585,17 +3630,15 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relation = new Document($relation); } if ($relation instanceof Document) { - $oldDocument = Authorization::skip(fn () => $dbForProject->getDocument( - 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), - $relation->getId() - )); + $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); + }); + $relation->removeAttribute('$collectionId'); $relation->removeAttribute('$databaseId'); // Attribute $collection is required for Utopia. - $relation->setAttribute( - '$collection', - 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId() - ); + $relation->setAttribute('$collection', 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); if ($oldDocument->isEmpty()) { if (isset($relation['$id']) && $relation['$id'] === 'unique()') { @@ -3617,14 +3660,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); try { - $document = $dbForProject->withRequestTimestamp( - $requestTimestamp, - fn () => $dbForProject->updateDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $document->getId(), - $newDocument - ) - ); + $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); + }); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (DuplicateException) { @@ -3636,7 +3675,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3656,9 +3695,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3720,6 +3760,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3728,32 +3769,36 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Read permission should not be required for delete - $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); } try { - $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) { - $dbForProject->deleteDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $documentId - ); + $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); } catch (NotFoundException $e) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3773,9 +3818,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($related as $relation) { if ($relation instanceof Document) { diff --git a/app/init.php b/app/init.php index c6847610c2..d5b8e4b235 100644 --- a/app/init.php +++ b/app/init.php @@ -232,6 +232,8 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; +const METRIC_DATABASE_API_READ = 'database.api.events.read'; +const METRIC_DATABASE_API_WRITE = 'database.api.events.write'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From 025c9aeeb297f6a0a96df401c6d0281321d17fb5 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:53:45 +0200 Subject: [PATCH 09/48] database crud usage addition --- src/Appwrite/Event/Usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index d2e302f82b..89e900d2ab 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -69,7 +69,7 @@ class Usage extends Event 'reduce' => $this->reduce, 'metrics' => $this->metrics, ]); - var_dump($this->metrics); + $this->metrics = []; return $result; From 8e17fc69b371861ee04da7e98a99fac294b42588 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:57:29 +0200 Subject: [PATCH 10/48] database crud usage addition --- app/controllers/api/databases.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 80bc72c4b7..b98df1382f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2898,8 +2898,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); From dd13e8b3cf705be22540b93ff3be493a771fc399 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 18:02:19 +0200 Subject: [PATCH 11/48] database crud usage addition --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b98df1382f..4275ab52ae 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3219,7 +3219,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function() use ($dbForProject, $queueForUsage, $database, $relatedCollectionId) { + $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); From 26bac5a2ea05cc76b49a79f848953a72d4125ade Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 30 Dec 2024 09:00:52 +0000 Subject: [PATCH 12/48] Remove firebase OAuth API endpoints --- app/config/specs/open-api3-latest-client.json | 6 +- .../specs/open-api3-latest-console.json | 415 +++------------ app/config/specs/open-api3-latest-server.json | 102 ++-- app/config/specs/swagger2-latest-client.json | 6 +- app/config/specs/swagger2-latest-console.json | 430 +++------------- app/config/specs/swagger2-latest-server.json | 102 ++-- app/controllers/api/migrations.php | 474 +----------------- src/Appwrite/Auth/OAuth2/Firebase.php | 389 -------------- 8 files changed, 239 insertions(+), 1685 deletions(-) delete mode 100644 src/Appwrite/Auth/OAuth2/Firebase.php diff --git a/app/config/specs/open-api3-latest-client.json b/app/config/specs/open-api3-latest-client.json index d948a101f2..af7303c985 100644 --- a/app/config/specs/open-api3-latest-client.json +++ b/app/config/specs/open-api3-latest-client.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -5766,7 +5766,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -5851,7 +5851,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index fb01509ecd..6878909f64 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -9488,7 +9488,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -10145,7 +10146,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -13673,7 +13675,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -13751,7 +13753,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -13897,7 +13899,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -14045,7 +14047,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -14202,7 +14204,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -14361,7 +14363,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -14472,7 +14474,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -14586,7 +14588,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -14641,7 +14643,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -14705,7 +14707,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -14782,7 +14784,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -14859,7 +14861,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -14937,7 +14939,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -15044,7 +15046,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -15154,7 +15156,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -15241,7 +15243,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -15331,7 +15333,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -15448,7 +15450,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -15568,7 +15570,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -15665,7 +15667,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -15765,7 +15767,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -15872,7 +15874,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -15982,7 +15984,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -16127,7 +16129,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -16274,7 +16276,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -16371,7 +16373,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -16471,7 +16473,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -16568,7 +16570,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -16668,7 +16670,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -16765,7 +16767,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -16865,7 +16867,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -16962,7 +16964,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -17062,7 +17064,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -17117,7 +17119,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -17181,7 +17183,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -17258,7 +17260,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -17335,7 +17337,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -17411,7 +17413,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -17496,7 +17498,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -17558,7 +17560,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -17637,7 +17639,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -17701,7 +17703,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -17778,7 +17780,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -17864,7 +17866,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -17956,7 +17958,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -18021,7 +18023,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -18098,7 +18100,7 @@ }, "x-appwrite": { "method": "list", - "weight": 339, + "weight": 338, "cookies": false, "type": "", "deprecated": false, @@ -18264,7 +18266,7 @@ }, "x-appwrite": { "method": "getAppwriteReport", - "weight": 341, + "weight": 340, "cookies": false, "type": "", "deprecated": false, @@ -18339,7 +18341,7 @@ }, "\/migrations\/firebase": { "post": { - "summary": "Migrate Firebase data (Service Account)", + "summary": "Migrate Firebase data", "operationId": "migrationsCreateFirebaseMigration", "tags": [ "migrations" @@ -18359,7 +18361,7 @@ }, "x-appwrite": { "method": "createFirebaseMigration", - "weight": 336, + "weight": 335, "cookies": false, "type": "", "deprecated": false, @@ -18415,177 +18417,6 @@ } } }, - "\/migrations\/firebase\/deauthorize": { - "get": { - "summary": "Revoke Appwrite's authorization to access Firebase projects", - "operationId": "migrationsDeleteFirebaseAuth", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "File" - } - }, - "x-appwrite": { - "method": "deleteFirebaseAuth", - "weight": 347, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/delete-firebase-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, - "\/migrations\/firebase\/oauth": { - "post": { - "summary": "Migrate Firebase data (OAuth)", - "operationId": "migrationsCreateFirebaseOAuthMigration", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "202": { - "description": "Migration", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/migration" - } - } - } - } - }, - "x-appwrite": { - "method": "createFirebaseOAuthMigration", - "weight": 335, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/create-firebase-o-auth-migration.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "requestBody": { - "content": { - "application\/json": { - "schema": { - "type": "object", - "properties": { - "resources": { - "type": "array", - "description": "List of resources to migrate", - "x-example": null, - "items": { - "type": "string" - } - }, - "projectId": { - "type": "string", - "description": "Project ID of the Firebase Project", - "x-example": "" - } - }, - "required": [ - "resources", - "projectId" - ] - } - } - } - } - } - }, - "\/migrations\/firebase\/projects": { - "get": { - "summary": "List Firebase projects", - "operationId": "migrationsListFirebaseProjects", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migrations Firebase Projects List", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/firebaseProjectList" - } - } - } - } - }, - "x-appwrite": { - "method": "listFirebaseProjects", - "weight": 346, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/list-firebase-projects.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.read", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, "\/migrations\/firebase\/report": { "get": { "summary": "Generate a report on Firebase data", @@ -18608,7 +18439,7 @@ }, "x-appwrite": { "method": "getFirebaseReport", - "weight": 342, + "weight": 341, "cookies": false, "type": "", "deprecated": false, @@ -18660,80 +18491,6 @@ ] } }, - "\/migrations\/firebase\/report\/oauth": { - "get": { - "summary": "Generate a report on Firebase data using OAuth", - "operationId": "migrationsGetFirebaseReportOAuth", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migration Report", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/migrationReport" - } - } - } - } - }, - "x-appwrite": { - "method": "getFirebaseReportOAuth", - "weight": 343, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/get-firebase-report-o-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase-report.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "parameters": [ - { - "name": "resources", - "description": "List of resources to migrate", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - }, - "in": "query" - }, - { - "name": "projectId", - "description": "Project ID", - "required": true, - "schema": { - "type": "string", - "x-example": "" - }, - "in": "query" - } - ] - } - }, "\/migrations\/nhost": { "post": { "summary": "Migrate NHost data", @@ -18756,7 +18513,7 @@ }, "x-appwrite": { "method": "createNHostMigration", - "weight": 338, + "weight": 337, "cookies": false, "type": "", "deprecated": false, @@ -18869,7 +18626,7 @@ }, "x-appwrite": { "method": "getNHostReport", - "weight": 349, + "weight": 343, "cookies": false, "type": "", "deprecated": false, @@ -19004,7 +18761,7 @@ }, "x-appwrite": { "method": "createSupabaseMigration", - "weight": 337, + "weight": 336, "cookies": false, "type": "", "deprecated": false, @@ -19111,7 +18868,7 @@ }, "x-appwrite": { "method": "getSupabaseReport", - "weight": 348, + "weight": 342, "cookies": false, "type": "", "deprecated": false, @@ -19237,7 +18994,7 @@ }, "x-appwrite": { "method": "get", - "weight": 340, + "weight": 339, "cookies": false, "type": "", "deprecated": false, @@ -19297,7 +19054,7 @@ }, "x-appwrite": { "method": "retry", - "weight": 350, + "weight": 344, "cookies": false, "type": "", "deprecated": false, @@ -19350,7 +19107,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 351, + "weight": 345, "cookies": false, "type": "", "deprecated": false, @@ -32749,30 +32506,6 @@ "migrations" ] }, - "firebaseProjectList": { - "description": "Migrations Firebase Projects List", - "type": "object", - "properties": { - "total": { - "type": "integer", - "description": "Total number of projects documents that matched your query.", - "x-example": 5, - "format": "int32" - }, - "projects": { - "type": "array", - "description": "List of projects.", - "items": { - "$ref": "#\/components\/schemas\/firebaseProject" - }, - "x-example": "" - } - }, - "required": [ - "total", - "projects" - ] - }, "specificationList": { "description": "Specifications List", "type": "object", @@ -35109,7 +34842,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { @@ -38707,26 +38440,6 @@ "size", "version" ] - }, - "firebaseProject": { - "description": "MigrationFirebaseProject", - "type": "object", - "properties": { - "projectId": { - "type": "string", - "description": "Project ID.", - "x-example": "my-project" - }, - "displayName": { - "type": "string", - "description": "Project display name.", - "x-example": "My Project" - } - }, - "required": [ - "projectId", - "displayName" - ] } }, "securitySchemes": { diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 1504322456..8900d6ddf2 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -8596,7 +8596,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -9019,7 +9020,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -12527,7 +12529,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -12606,7 +12608,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -12753,7 +12755,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -12902,7 +12904,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -13060,7 +13062,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -13220,7 +13222,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -13332,7 +13334,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -13447,7 +13449,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -13503,7 +13505,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -13568,7 +13570,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -13646,7 +13648,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -13724,7 +13726,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -13803,7 +13805,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -13911,7 +13913,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -14022,7 +14024,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -14110,7 +14112,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -14201,7 +14203,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -14319,7 +14321,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -14440,7 +14442,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -14538,7 +14540,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -14639,7 +14641,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -14747,7 +14749,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -14858,7 +14860,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -15004,7 +15006,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -15152,7 +15154,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -15250,7 +15252,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -15351,7 +15353,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -15449,7 +15451,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -15550,7 +15552,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -15648,7 +15650,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -15749,7 +15751,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -15847,7 +15849,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -15948,7 +15950,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -16004,7 +16006,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -16069,7 +16071,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -16147,7 +16149,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -16225,7 +16227,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -16302,7 +16304,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -16388,7 +16390,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -16451,7 +16453,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -16531,7 +16533,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -16596,7 +16598,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -16674,7 +16676,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -16761,7 +16763,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -16855,7 +16857,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -16921,7 +16923,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -25746,7 +25748,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { diff --git a/app/config/specs/swagger2-latest-client.json b/app/config/specs/swagger2-latest-client.json index 3ac70ffe28..77025ec042 100644 --- a/app/config/specs/swagger2-latest-client.json +++ b/app/config/specs/swagger2-latest-client.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -5981,7 +5981,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -6070,7 +6070,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 25c0c289b3..0ef937050c 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -9610,7 +9610,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -10286,7 +10287,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -13890,7 +13892,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -13967,7 +13969,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -14127,7 +14129,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -14284,7 +14286,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -14459,7 +14461,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -14631,7 +14633,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -14751,7 +14753,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -14869,7 +14871,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -14928,7 +14930,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -14992,7 +14994,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -15068,7 +15070,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -15144,7 +15146,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -15221,7 +15223,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -15338,7 +15340,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -15453,7 +15455,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -15546,7 +15548,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -15637,7 +15639,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -15766,7 +15768,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -15893,7 +15895,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -15998,7 +16000,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -16101,7 +16103,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -16218,7 +16220,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -16333,7 +16335,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -16494,7 +16496,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -16652,7 +16654,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -16757,7 +16759,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -16860,7 +16862,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -16965,7 +16967,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -17068,7 +17070,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -17173,7 +17175,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -17276,7 +17278,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -17381,7 +17383,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -17484,7 +17486,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -17543,7 +17545,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -17607,7 +17609,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -17683,7 +17685,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -17759,7 +17761,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -17834,7 +17836,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -17926,7 +17928,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -17988,7 +17990,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -18071,7 +18073,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -18135,7 +18137,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -18211,7 +18213,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -18294,7 +18296,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -18386,7 +18388,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -18453,7 +18455,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -18528,7 +18530,7 @@ }, "x-appwrite": { "method": "list", - "weight": 339, + "weight": 338, "cookies": false, "type": "", "deprecated": false, @@ -18699,7 +18701,7 @@ }, "x-appwrite": { "method": "getAppwriteReport", - "weight": 341, + "weight": 340, "cookies": false, "type": "", "deprecated": false, @@ -18767,7 +18769,7 @@ }, "\/migrations\/firebase": { "post": { - "summary": "Migrate Firebase data (Service Account)", + "summary": "Migrate Firebase data", "operationId": "migrationsCreateFirebaseMigration", "consumes": [ "application\/json" @@ -18789,7 +18791,7 @@ }, "x-appwrite": { "method": "createFirebaseMigration", - "weight": 336, + "weight": 335, "cookies": false, "type": "", "deprecated": false, @@ -18847,192 +18849,6 @@ ] } }, - "\/migrations\/firebase\/deauthorize": { - "get": { - "summary": "Revoke Appwrite's authorization to access Firebase projects", - "operationId": "migrationsDeleteFirebaseAuth", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "File", - "schema": { - "type": "file" - } - } - }, - "x-appwrite": { - "method": "deleteFirebaseAuth", - "weight": 347, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/delete-firebase-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, - "\/migrations\/firebase\/oauth": { - "post": { - "summary": "Migrate Firebase data (OAuth)", - "operationId": "migrationsCreateFirebaseOAuthMigration", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "202": { - "description": "Migration", - "schema": { - "$ref": "#\/definitions\/migration" - } - } - }, - "x-appwrite": { - "method": "createFirebaseOAuthMigration", - "weight": 335, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/create-firebase-o-auth-migration.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "parameters": [ - { - "name": "payload", - "in": "body", - "schema": { - "type": "object", - "properties": { - "resources": { - "type": "array", - "description": "List of resources to migrate", - "default": null, - "x-example": null, - "items": { - "type": "string" - } - }, - "projectId": { - "type": "string", - "description": "Project ID of the Firebase Project", - "default": null, - "x-example": "" - } - }, - "required": [ - "resources", - "projectId" - ] - } - } - ] - } - }, - "\/migrations\/firebase\/projects": { - "get": { - "summary": "List Firebase projects", - "operationId": "migrationsListFirebaseProjects", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migrations Firebase Projects List", - "schema": { - "$ref": "#\/definitions\/firebaseProjectList" - } - } - }, - "x-appwrite": { - "method": "listFirebaseProjects", - "weight": 346, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/list-firebase-projects.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.read", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, "\/migrations\/firebase\/report": { "get": { "summary": "Generate a report on Firebase data", @@ -19057,7 +18873,7 @@ }, "x-appwrite": { "method": "getFirebaseReport", - "weight": 342, + "weight": 341, "cookies": false, "type": "", "deprecated": false, @@ -19106,79 +18922,6 @@ ] } }, - "\/migrations\/firebase\/report\/oauth": { - "get": { - "summary": "Generate a report on Firebase data using OAuth", - "operationId": "migrationsGetFirebaseReportOAuth", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migration Report", - "schema": { - "$ref": "#\/definitions\/migrationReport" - } - } - }, - "x-appwrite": { - "method": "getFirebaseReportOAuth", - "weight": 343, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/get-firebase-report-o-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase-report.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "parameters": [ - { - "name": "resources", - "description": "List of resources to migrate", - "required": true, - "type": "array", - "collectionFormat": "multi", - "items": { - "type": "string" - }, - "in": "query" - }, - { - "name": "projectId", - "description": "Project ID", - "required": true, - "type": "string", - "x-example": "", - "in": "query" - } - ] - } - }, "\/migrations\/nhost": { "post": { "summary": "Migrate NHost data", @@ -19203,7 +18946,7 @@ }, "x-appwrite": { "method": "createNHostMigration", - "weight": 338, + "weight": 337, "cookies": false, "type": "", "deprecated": false, @@ -19326,7 +19069,7 @@ }, "x-appwrite": { "method": "getNHostReport", - "weight": 349, + "weight": 343, "cookies": false, "type": "", "deprecated": false, @@ -19448,7 +19191,7 @@ }, "x-appwrite": { "method": "createSupabaseMigration", - "weight": 337, + "weight": 336, "cookies": false, "type": "", "deprecated": false, @@ -19564,7 +19307,7 @@ }, "x-appwrite": { "method": "getSupabaseReport", - "weight": 348, + "weight": 342, "cookies": false, "type": "", "deprecated": false, @@ -19679,7 +19422,7 @@ }, "x-appwrite": { "method": "get", - "weight": 340, + "weight": 339, "cookies": false, "type": "", "deprecated": false, @@ -19739,7 +19482,7 @@ }, "x-appwrite": { "method": "retry", - "weight": 350, + "weight": 344, "cookies": false, "type": "", "deprecated": false, @@ -19794,7 +19537,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 351, + "weight": 345, "cookies": false, "type": "", "deprecated": false, @@ -33250,31 +32993,6 @@ "migrations" ] }, - "firebaseProjectList": { - "description": "Migrations Firebase Projects List", - "type": "object", - "properties": { - "total": { - "type": "integer", - "description": "Total number of projects documents that matched your query.", - "x-example": 5, - "format": "int32" - }, - "projects": { - "type": "array", - "description": "List of projects.", - "items": { - "type": "object", - "$ref": "#\/definitions\/firebaseProject" - }, - "x-example": "" - } - }, - "required": [ - "total", - "projects" - ] - }, "specificationList": { "description": "Specifications List", "type": "object", @@ -35618,7 +35336,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { @@ -39274,26 +38992,6 @@ "size", "version" ] - }, - "firebaseProject": { - "description": "MigrationFirebaseProject", - "type": "object", - "properties": { - "projectId": { - "type": "string", - "description": "Project ID.", - "x-example": "my-project" - }, - "displayName": { - "type": "string", - "description": "Project display name.", - "x-example": "My Project" - } - }, - "required": [ - "projectId", - "displayName" - ] } }, "externalDocs": { diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 68bcf268fb..f5b1f24a22 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -8720,7 +8720,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -9166,7 +9167,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -12752,7 +12754,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -12830,7 +12832,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -12991,7 +12993,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -13149,7 +13151,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -13325,7 +13327,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -13498,7 +13500,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -13619,7 +13621,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -13738,7 +13740,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -13798,7 +13800,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -13863,7 +13865,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -13940,7 +13942,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -14017,7 +14019,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -14095,7 +14097,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -14213,7 +14215,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -14329,7 +14331,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -14423,7 +14425,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -14515,7 +14517,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -14645,7 +14647,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -14773,7 +14775,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -14879,7 +14881,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -14983,7 +14985,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -15101,7 +15103,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -15217,7 +15219,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -15379,7 +15381,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -15538,7 +15540,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -15644,7 +15646,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -15748,7 +15750,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -15854,7 +15856,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -15958,7 +15960,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -16064,7 +16066,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -16168,7 +16170,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -16274,7 +16276,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -16378,7 +16380,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -16438,7 +16440,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -16503,7 +16505,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -16580,7 +16582,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -16657,7 +16659,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -16733,7 +16735,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -16826,7 +16828,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -16889,7 +16891,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -16973,7 +16975,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -17038,7 +17040,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -17115,7 +17117,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -17199,7 +17201,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -17293,7 +17295,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -17361,7 +17363,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -26233,7 +26235,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 3abd1db016..29e46505f5 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -1,17 +1,12 @@ dynamic($migration, Response::MODEL_MIGRATION); }); -App::post('/v1/migrations/firebase/oauth') - ->groups(['api', 'migrations']) - ->desc('Migrate Firebase data (OAuth)') - ->label('scope', 'migrations.write') - ->label('event', 'migrations.[migrationId].create') - ->label('audits.event', 'migration.create') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'migrations') - ->label('sdk.method', 'createFirebaseOAuthMigration') - ->label('sdk.description', '/docs/references/migrations/migration-firebase.md') - ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_MIGRATION) - ->param('resources', [], new ArrayList(new WhiteList(Firebase::getSupportedResources())), 'List of resources to migrate') - ->param('projectId', '', new Text(65536), 'Project ID of the Firebase Project') - ->inject('response') - ->inject('dbForProject') - ->inject('dbForPlatform') - ->inject('project') - ->inject('user') - ->inject('queueForEvents') - ->inject('queueForMigrations') - ->inject('request') - ->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForPlatform, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations, Request $request) { - $firebase = new OAuth2Firebase( - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), - $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' - ); - - $identity = $dbForPlatform->findOne('identities', [ - Query::equal('provider', ['firebase']), - Query::equal('userInternalId', [$user->getInternalId()]), - ]); - if ($identity->isEmpty()) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - $accessToken = $identity->getAttribute('providerAccessToken'); - $refreshToken = $identity->getAttribute('providerRefreshToken'); - $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); - - $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); - if ($isExpired) { - $firebase->refreshTokens($refreshToken); - - $accessToken = $firebase->getAccessToken(''); - $refreshToken = $firebase->getRefreshToken(''); - - $verificationId = $firebase->getUserID($accessToken); - - if (empty($verificationId)) { - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); - } - - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); - - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); - } - - if ($identity->getAttribute('secrets')) { - $serviceAccount = $identity->getAttribute('secrets'); - } else { - $firebase->cleanupServiceAccounts($accessToken, $projectId); - $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId); - $identity = $identity - ->setAttribute('secrets', json_encode($serviceAccount)); - - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); - } - - $migration = $dbForProject->createDocument('migrations', new Document([ - '$id' => ID::unique(), - 'status' => 'pending', - 'stage' => 'init', - 'source' => Firebase::getName(), - 'destination' => Appwrite::getName(), - 'credentials' => [ - 'serviceAccount' => json_encode($serviceAccount), - ], - 'resources' => $resources, - 'statusCounters' => '{}', - 'resourceData' => '{}', - 'errors' => [] - ])); - - $queueForEvents->setParam('migrationId', $migration->getId()); - - // Trigger Transfer - $queueForMigrations - ->setMigration($migration) - ->setProject($project) - ->setUser($user) - ->trigger(); - - $response - ->setStatusCode(Response::STATUS_CODE_ACCEPTED) - ->dynamic($migration, Response::MODEL_MIGRATION); - }); - App::post('/v1/migrations/firebase') ->groups(['api', 'migrations']) - ->desc('Migrate Firebase data (Service Account)') + ->desc('Migrate Firebase data') ->label('scope', 'migrations.write') ->label('event', 'migrations.[migrationId].create') ->label('audits.event', 'migration.create') @@ -547,368 +437,6 @@ App::get('/v1/migrations/firebase/report') ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT); }); -App::get('/v1/migrations/firebase/report/oauth') - ->groups(['api', 'migrations']) - ->desc('Generate a report on Firebase data using OAuth') - ->label('scope', 'migrations.write') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'migrations') - ->label('sdk.method', 'getFirebaseReportOAuth') - ->label('sdk.description', '/docs/references/migrations/migration-firebase-report.md') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT) - ->param('resources', [], new ArrayList(new WhiteList(Firebase::getSupportedResources())), 'List of resources to migrate') - ->param('projectId', '', new Text(65536), 'Project ID') - ->inject('response') - ->inject('request') - ->inject('user') - ->inject('dbForPlatform') - ->action(function (array $resources, string $projectId, Response $response, Request $request, Document $user, Database $dbForPlatform) { - $firebase = new OAuth2Firebase( - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), - $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' - ); - - $identity = $dbForPlatform->findOne('identities', [ - Query::equal('provider', ['firebase']), - Query::equal('userInternalId', [$user->getInternalId()]), - ]); - - if ($identity->isEmpty()) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - $accessToken = $identity->getAttribute('providerAccessToken'); - $refreshToken = $identity->getAttribute('providerRefreshToken'); - $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); - - if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - if (System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); - if ($isExpired) { - $firebase->refreshTokens($refreshToken); - - $accessToken = $firebase->getAccessToken(''); - $refreshToken = $firebase->getRefreshToken(''); - - $verificationId = $firebase->getUserID($accessToken); - - if (empty($verificationId)) { - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); - } - - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); - - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); - } - - // Get Service Account - if ($identity->getAttribute('secrets')) { - $serviceAccount = $identity->getAttribute('secrets'); - } else { - $firebase->cleanupServiceAccounts($accessToken, $projectId); - $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId); - $identity = $identity - ->setAttribute('secrets', json_encode($serviceAccount)); - - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); - } - - $firebase = new Firebase($serviceAccount); - - try { - $report = $firebase->report($resources); - } catch (\Throwable $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage()); - } - - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT); - }); - -App::get('/v1/migrations/firebase/connect') - ->desc('Authorize with Firebase') - ->groups(['api', 'migrations']) - ->label('scope', 'migrations.write') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'migrations') - ->label('sdk.method', 'createFirebaseAuth') - ->label('sdk.description', '') - ->label('sdk.response.code', Response::STATUS_CODE_MOVED_PERMANENTLY) - ->label('sdk.response.type', Response::CONTENT_TYPE_HTML) - ->label('sdk.methodType', 'webAuth') - ->label('sdk.hide', true) - ->param('redirect', '', fn ($clients) => new Host($clients), 'URL to redirect back to your Firebase authorization. Only console hostnames are allowed.', true, ['clients']) - ->param('projectId', '', new UID(), 'Project ID') - ->inject('response') - ->inject('request') - ->inject('user') - ->inject('dbForPlatform') - ->action(function (string $redirect, string $projectId, Response $response, Request $request, Document $user, Database $dbForPlatform) { - $state = \json_encode([ - 'projectId' => $projectId, - 'redirect' => $redirect, - ]); - - $prefs = $user->getAttribute('prefs', []); - $prefs['migrationState'] = $state; - $user->setAttribute('prefs', $prefs); - $dbForPlatform->updateDocument('users', $user->getId(), $user); - - $oauth2 = new OAuth2Firebase( - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), - $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' - ); - $url = $oauth2->getLoginURL(); - - $response - ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') - ->addHeader('Pragma', 'no-cache') - ->redirect($url); - }); - -App::get('/v1/migrations/firebase/redirect') - ->desc('Capture and receive data on Firebase authorization') - ->groups(['api', 'migrations']) - ->label('scope', 'public') - ->label('error', __DIR__ . '/../../views/general/error.phtml') - ->param('code', '', new Text(2048), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true) - ->inject('user') - ->inject('project') - ->inject('request') - ->inject('response') - ->inject('dbForPlatform') - ->action(function (string $code, Document $user, Document $project, Request $request, Response $response, Database $dbForPlatform) { - $state = $user['prefs']['migrationState'] ?? '{}'; - $prefs['migrationState'] = ''; - $user->setAttribute('prefs', $prefs); - $dbForPlatform->updateDocument('users', $user->getId(), $user); - - if (empty($state)) { - throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Installation requests from organisation members for the Appwrite Google App are currently unsupported.'); - } - - $state = \json_decode($state, true); - $redirect = $state['redirect'] ?? ''; - $projectId = $state['projectId'] ?? ''; - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if (empty($redirect)) { - $redirect = $request->getProtocol() . '://' . $request->getHostname() . '/console/project-$projectId/settings/migrations'; - } - - if ($project->isEmpty()) { - $response - ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') - ->addHeader('Pragma', 'no-cache') - ->redirect($redirect); - - return; - } - - // OAuth Authroization - if (!empty($code)) { - $oauth2 = new OAuth2Firebase( - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), - $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' - ); - - $accessToken = $oauth2->getAccessToken($code); - $refreshToken = $oauth2->getRefreshToken($code); - $accessTokenExpiry = $oauth2->getAccessTokenExpiry($code); - $email = $oauth2->getUserEmail($accessToken); - $oauth2ID = $oauth2->getUserID($accessToken); - - if (empty($accessToken)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to get access token.'); - } - - if (empty($refreshToken)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to get refresh token.'); - } - - if (empty($accessTokenExpiry)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to get access token expiry.'); - } - - // Makes sure this email is not already used in another identity - $identity = $dbForPlatform->findOne('identities', [ - Query::equal('providerEmail', [$email]), - ]); - - if (!$identity->isEmpty()) { - if ($identity->getAttribute('userInternalId', '') !== $user->getInternalId()) { - throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); - } - } - - if (!$identity->isEmpty()) { - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$accessTokenExpiry)); - - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); - } else { - $identity = $dbForPlatform->createDocument('identities', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::user($user->getId())), - Permission::delete(Role::user($user->getId())), - ], - 'userInternalId' => $user->getInternalId(), - 'userId' => $user->getId(), - 'provider' => 'firebase', - 'providerUid' => $oauth2ID, - 'providerEmail' => $email, - 'providerAccessToken' => $accessToken, - 'providerRefreshToken' => $refreshToken, - 'providerAccessTokenExpiry' => DateTime::addSeconds(new \DateTime(), (int)$accessTokenExpiry), - ])); - } - } else { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Missing OAuth2 code.'); - } - - $response - ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') - ->addHeader('Pragma', 'no-cache') - ->redirect($redirect); - }); - -App::get('/v1/migrations/firebase/projects') - ->desc('List Firebase projects') - ->groups(['api', 'migrations']) - ->label('scope', 'migrations.read') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'migrations') - ->label('sdk.method', 'listFirebaseProjects') - ->label('sdk.description', '') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_MIGRATION_FIREBASE_PROJECT_LIST) - ->inject('user') - ->inject('response') - ->inject('project') - ->inject('dbForPlatform') - ->inject('request') - ->action(function (Document $user, Response $response, Document $project, Database $dbForPlatform, Request $request) { - $firebase = new OAuth2Firebase( - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), - System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), - $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' - ); - - $identity = $dbForPlatform->findOne('identities', [ - Query::equal('provider', ['firebase']), - Query::equal('userInternalId', [$user->getInternalId()]), - ]); - - if ($identity->isEmpty()) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - $accessToken = $identity->getAttribute('providerAccessToken'); - $refreshToken = $identity->getAttribute('providerRefreshToken'); - $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); - - if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - if (System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - try { - $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); - if ($isExpired) { - try { - $firebase->refreshTokens($refreshToken); - } catch (\Throwable $e) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - $accessToken = $firebase->getAccessToken(''); - $refreshToken = $firebase->getRefreshToken(''); - - $verificationId = $firebase->getUserID($accessToken); - - if (empty($verificationId)) { - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); - } - - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); - - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); - } - - $projects = $firebase->getProjects($accessToken); - - $output = []; - foreach ($projects as $project) { - $output[] = [ - 'displayName' => $project['displayName'], - 'projectId' => $project['projectId'], - ]; - } - } catch (\Throwable $e) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - $response->dynamic(new Document([ - 'projects' => $output, - 'total' => count($output), - ]), Response::MODEL_MIGRATION_FIREBASE_PROJECT_LIST); - }); - -App::get('/v1/migrations/firebase/deauthorize') - ->desc('Revoke Appwrite\'s authorization to access Firebase projects') - ->groups(['api', 'migrations']) - ->label('scope', 'migrations.write') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'migrations') - ->label('sdk.method', 'deleteFirebaseAuth') - ->label('sdk.description', '') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->inject('user') - ->inject('response') - ->inject('dbForPlatform') - ->action(function (Document $user, Response $response, Database $dbForPlatform) { - $identity = $dbForPlatform->findOne('identities', [ - Query::equal('provider', ['firebase']), - Query::equal('userInternalId', [$user->getInternalId()]), - ]); - - if ($identity->isEmpty()) { - throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Not authenticated with Firebase'); //TODO: Replace with USER_IDENTITY_NOT_FOUND - } - - $dbForPlatform->deleteDocument('identities', $identity->getId()); - - $response->noContent(); - }); - App::get('/v1/migrations/supabase/report') ->groups(['api', 'migrations']) ->desc('Generate a report on Supabase Data') diff --git a/src/Appwrite/Auth/OAuth2/Firebase.php b/src/Appwrite/Auth/OAuth2/Firebase.php deleted file mode 100644 index 0e2859e32c..0000000000 --- a/src/Appwrite/Auth/OAuth2/Firebase.php +++ /dev/null @@ -1,389 +0,0 @@ - 'offline', - 'client_id' => $this->appID, - 'redirect_uri' => $this->callback, - 'scope' => \implode(' ', $this->getScopes()), - 'state' => \json_encode($this->state), - 'response_type' => 'code', - 'prompt' => 'consent', - ]); - } - - /** - * @param string $code - * - * @return array - */ - protected function getTokens(string $code): array - { - if (empty($this->tokens)) { - $response = $this->request( - 'POST', - 'https://oauth2.googleapis.com/token', - [], - \http_build_query([ - 'client_id' => $this->appID, - 'redirect_uri' => $this->callback, - 'client_secret' => $this->appSecret, - 'code' => $code, - 'grant_type' => 'authorization_code' - ]) - ); - - $this->tokens = \json_decode($response, true); - } - - return $this->tokens; - } - - /** - * @param string $refreshToken - * - * @return array - */ - public function refreshTokens(string $refreshToken): array - { - $response = $this->request( - 'POST', - 'https://oauth2.googleapis.com/token', - [], - \http_build_query([ - 'client_id' => $this->appID, - 'client_secret' => $this->appSecret, - 'grant_type' => 'refresh_token', - 'refresh_token' => $refreshToken - ]) - ); - - $output = []; - \parse_str($response, $output); - $this->tokens = $output; - - if (empty($this->tokens['refresh_token'])) { - $this->tokens['refresh_token'] = $refreshToken; - } - - return $this->tokens; - } - - - /** - * @param string $accessToken - * - * @return string - */ - public function getUserID(string $accessToken): string - { - $user = $this->getUser($accessToken); - - return $user['id'] ?? ''; - } - - /** - * @param string $accessToken - * - * @return string - */ - public function getUserEmail(string $accessToken): string - { - $user = $this->getUser($accessToken); - - return $user['email'] ?? ''; - } - - - /** - * Check if the OAuth email is verified - * - * @link https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user - * - * @param string $accessToken - * - * @return bool - */ - public function isEmailVerified(string $accessToken): bool - { - $user = $this->getUser($accessToken); - - if ($user['verified'] ?? false) { - return true; - } - - return false; - } - - /** - * @param string $accessToken - * - * @return string - */ - public function getUserName(string $accessToken): string - { - $user = $this->getUser($accessToken); - - return $user['name'] ?? ''; - } - - /** - * @param string $accessToken - * - * @return array - */ - protected function getUser(string $accessToken) - { - if (empty($this->user)) { - $response = $this->request( - 'GET', - 'https://www.googleapis.com/oauth2/v1/userinfo?access_token=' . \urlencode($accessToken), - [], - ); - - $this->user = \json_decode($response, true); - } - - return $this->user; - } - - public function getProjects(string $accessToken): array - { - $projects = $this->request('GET', 'https://firebase.googleapis.com/v1beta1/projects', ['Authorization: Bearer ' . \urlencode($accessToken)]); - - $projects = \json_decode($projects, true); - - return $projects['results']; - } - - /* - Be careful with the setIAMPolicy method, it will overwrite all existing policies - **/ - public function assignIAMRole(string $accessToken, string $email, string $projectId, array $role) - { - // Get IAM Roles - $iamRoles = $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':getIamPolicy', [ - 'Authorization: Bearer ' . \urlencode($accessToken), - 'Content-Type: application/json' - ]); - - $iamRoles = \json_decode($iamRoles, true); - - $iamRoles['bindings'][] = [ - 'role' => $role['name'], - 'members' => [ - 'serviceAccount:' . $email - ] - ]; - - // Set IAM Roles - $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':setIamPolicy', [ - 'Authorization: Bearer ' . \urlencode($accessToken), - 'Content-Type: application/json' - ], \json_encode([ - 'policy' => $iamRoles - ])); - } - - private function generateRandomString($length = 10): string - { - $characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; - $charactersLength = strlen($characters); - $randomString = ''; - for ($i = 0; $i < $length; $i++) { - $randomString .= $characters[random_int(0, $charactersLength - 1)]; - } - return $randomString; - } - - private function createCustomRole(string $accessToken, string $projectId): array - { - // Check if role already exists - try { - $role = $this->request('GET', 'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/appwriteMigrations', [ - 'Content-Type: application/json', - 'Authorization: Bearer ' . \urlencode($accessToken), - ]); - - $role = \json_decode($role, true); - - return $role; - } catch (\Throwable $e) { - if ($e->getCode() !== 404) { - throw $e; - } - } - - // Create role if doesn't exist or isn't correct - $role = $this->request( - 'POST', - 'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/', - [ - 'Content-Type: application/json', - 'Authorization: Bearer ' . \urlencode($accessToken), - ], - \json_encode( - [ - 'roleId' => 'appwriteMigrations', - 'role' => [ - 'title' => 'Appwrite Migrations', - 'description' => 'A helper role for Appwrite Migrations', - 'includedPermissions' => $this->iamPermissions, - 'stage' => 'GA' - ] - ] - ) - ); - - return json_decode($role, true); - } - - public function createServiceAccount(string $accessToken, string $projectId): array - { - // Create Service Account - $uid = $this->generateRandomString(); - - $response = $this->request( - 'POST', - 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts', - [ - 'Authorization: Bearer ' . \urlencode($accessToken), - 'Content-Type: application/json' - ], - \json_encode([ - 'accountId' => 'appwrite-' . $uid, - 'serviceAccount' => [ - 'displayName' => 'Appwrite Migrations ' . $uid - ] - ]) - ); - - $response = json_decode($response, true); - - // Create and assign IAM Roles - $role = $this->createCustomRole($accessToken, $projectId); - - \sleep(1); // Wait for IAM to propagate changes. - - $this->assignIAMRole($accessToken, $response['email'], $projectId, $role); - - // Create Service Account Key - $responseKey = $this->request( - 'POST', - 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $response['email'] . '/keys', - [ - 'Authorization: Bearer ' . \urlencode($accessToken), - 'Content-Type: application/json' - ] - ); - - $responseKey = json_decode($responseKey, true); - - return json_decode(base64_decode($responseKey['privateKeyData']), true); - } - - public function cleanupServiceAccounts(string $accessToken, string $projectId) - { - // List Service Accounts - $response = $this->request( - 'GET', - 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts', - [ - 'Authorization: Bearer ' . \urlencode($accessToken), - 'Content-Type: application/json' - ] - ); - - $response = json_decode($response, true); - - if (empty($response['accounts'])) { - return false; - } - - foreach ($response['accounts'] as $account) { - if (strpos($account['email'], 'appwrite-') !== false) { - $this->request( - 'DELETE', - 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $account['email'], - [ - 'Authorization: Bearer ' . \urlencode($accessToken), - 'Content-Type: application/json' - ] - ); - } - } - - return true; - } -} From dd0658de1999e7b00c2bf4135507e919369039fa Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 30 Dec 2024 11:26:11 +0000 Subject: [PATCH 13/48] chore: bump versions --- app/config/platforms.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/platforms.php b/app/config/platforms.php index 1470bd1364..167e55651c 100644 --- a/app/config/platforms.php +++ b/app/config/platforms.php @@ -59,7 +59,7 @@ return [ [ 'key' => 'flutter', 'name' => 'Flutter', - 'version' => '13.1.0', + 'version' => '13.1.1', 'url' => 'https://github.com/appwrite/sdk-for-flutter', 'package' => 'https://pub.dev/packages/appwrite', 'enabled' => true, @@ -134,7 +134,7 @@ return [ [ 'key' => 'react-native', 'name' => 'React Native', - 'version' => '0.5.0', + 'version' => '0.6.0', 'url' => 'https://github.com/appwrite/sdk-for-react-native', 'package' => 'https://npmjs.com/package/react-native-appwrite', 'enabled' => true, From 8c7520329868b892e3d9984bba54cd50b4581ee0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 30 Dec 2024 11:59:19 +0000 Subject: [PATCH 14/48] chore: bump sdk generator --- composer.json | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index d8bee998c3..e44dd0bffc 100644 --- a/composer.json +++ b/composer.json @@ -84,7 +84,7 @@ }, "require-dev": { "ext-fileinfo": "*", - "appwrite/sdk-generator": "0.39.27", + "appwrite/sdk-generator": "0.39.28", "phpunit/phpunit": "9.5.20", "swoole/ide-helper": "5.1.2", "textalk/websocket": "1.5.7", diff --git a/composer.lock b/composer.lock index 419201d5f7..ce5f358619 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9712ceefbca8a06dfcf5e003f9c92171", + "content-hash": "6b136b5490c0d5d331eac0d70bb3e198", "packages": [ { "name": "adhocore/jwt", @@ -4807,16 +4807,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.27", + "version": "0.39.28", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849" + "reference": "6ff467858fe418e364460da905139216570a5d5e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/27d8ecde30e40cbfe1124cc0430c406d3e144849", - "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6ff467858fe418e364460da905139216570a5d5e", + "reference": "6ff467858fe418e364460da905139216570a5d5e", "shasum": "" }, "require": { @@ -4852,9 +4852,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.27" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.28" }, - "time": "2024-12-16T11:32:02+00:00" + "time": "2024-12-30T11:17:25+00:00" }, { "name": "doctrine/annotations", From d3b764a9e2e718b93a845979deca7a29ddeb8bd4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 30 Dec 2024 15:10:27 +0000 Subject: [PATCH 15/48] Set base specification CPUs to 0.5 again --- app/config/runtimes/specifications.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/runtimes/specifications.php b/app/config/runtimes/specifications.php index 68880f4d4e..d3625db8a2 100644 --- a/app/config/runtimes/specifications.php +++ b/app/config/runtimes/specifications.php @@ -6,7 +6,7 @@ return [ Specification::S_05VCPU_512MB => [ 'slug' => Specification::S_05VCPU_512MB, 'memory' => 512, - 'cpus' => 1 // TODO: revert this, it's a temporary change to test function performance. + 'cpus' => 0.5 ], Specification::S_1VCPU_512MB => [ 'slug' => Specification::S_1VCPU_512MB, From 0f2985cec1a2e80fc325b59824eedad64519b5f2 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:19:09 +0200 Subject: [PATCH 16/48] database crud usage addition --- app/controllers/api/databases.php | 56 +++++++++++++++---------------- app/init.php | 4 +-- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 4275ab52ae..e51201a467 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2886,7 +2886,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('databases', $databaseId); }); @@ -2899,7 +2899,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -2989,7 +2989,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3005,7 +3005,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } if ($relation instanceof Document) { $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); } ); @@ -3039,7 +3039,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3070,7 +3070,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3133,7 +3133,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3143,7 +3143,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3174,7 +3174,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3186,10 +3186,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); // Add $collectionId and $databaseId for all documents $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { @@ -3220,7 +3220,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3305,7 +3305,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3315,7 +3315,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3326,7 +3326,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3363,7 +3363,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3527,7 +3527,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('databases', $databaseId); }); @@ -3539,7 +3539,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3550,7 +3550,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3617,7 +3617,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3633,7 +3633,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } if ($relation instanceof Document) { $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); }); @@ -3663,7 +3663,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum try { $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); }); } catch (AuthorizationException) { @@ -3698,7 +3698,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3762,7 +3762,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3772,7 +3772,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3782,7 +3782,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu // Read permission should not be required for delete $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3792,7 +3792,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu try { $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); } catch (NotFoundException $e) { @@ -3821,7 +3821,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); diff --git a/app/init.php b/app/init.php index d5b8e4b235..d942e056db 100644 --- a/app/init.php +++ b/app/init.php @@ -232,8 +232,8 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASE_API_READ = 'database.api.events.read'; -const METRIC_DATABASE_API_WRITE = 'database.api.events.write'; +const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From 42589de95ef41114d247fdbceacce8ea3d708233 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:56:14 +0200 Subject: [PATCH 17/48] database crud usage addition --- app/controllers/api/databases.php | 170 +++++++++++++----------------- app/init.php | 2 + 2 files changed, 75 insertions(+), 97 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index e51201a467..ea17065d45 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2885,11 +2885,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is not allowed for creating new documents, try update instead'); } - $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('databases', $databaseId); - }); - + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -2898,10 +2894,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -2988,10 +2981,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($relations as &$relation) { @@ -3004,10 +2995,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relation = new Document($relation); } if ($relation instanceof Document) { - $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); - } + $current = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()) ); if ($current->isEmpty()) { @@ -3039,7 +3028,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3049,7 +3037,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3069,10 +3057,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($related as $relation) { @@ -3107,6 +3093,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); @@ -3133,7 +3121,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3143,7 +3130,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3173,10 +3159,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); - $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($cursorDocument->isEmpty()) { throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Document '{$documentId}' for the 'cursor' value not found."); @@ -3186,13 +3169,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { if ($document->isEmpty()) { return false; } @@ -3219,10 +3199,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); foreach ($relations as $index => $doc) { if ($doc instanceof Document) { @@ -3273,6 +3250,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ; + $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3305,7 +3287,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3314,10 +3295,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3326,7 +3304,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3338,7 +3315,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { if ($document->isEmpty()) { return; } @@ -3362,10 +3339,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($related as $relation) { @@ -3378,6 +3353,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3526,10 +3506,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DOCUMENT_MISSING_PAYLOAD); } - $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('databases', $databaseId); - }); + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3538,10 +3515,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3549,10 +3523,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ - $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); @@ -3594,7 +3565,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, $queueForUsage) { + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3616,10 +3587,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($relations as &$relation) { // If the relation is an array it can be either update or create a child document. @@ -3632,15 +3602,17 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relation = new Document($relation); } if ($relation instanceof Document) { - $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); - }); - + $oldDocument = Authorization::skip(fn () => $dbForProject->getDocument( + 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), + $relation->getId() + )); $relation->removeAttribute('$collectionId'); $relation->removeAttribute('$databaseId'); // Attribute $collection is required for Utopia. - $relation->setAttribute('$collection', 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + $relation->setAttribute( + '$collection', + 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId() + ); if ($oldDocument->isEmpty()) { if (isset($relation['$id']) && $relation['$id'] === 'unique()') { @@ -3662,10 +3634,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); try { - $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); - return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); - }); + $document = $dbForProject->withRequestTimestamp( + $requestTimestamp, + fn () => $dbForProject->updateDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $document->getId(), + $newDocument + ) + ); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (DuplicateException) { @@ -3677,7 +3653,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3697,10 +3673,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3712,6 +3687,11 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( @@ -3762,7 +3742,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3771,36 +3750,32 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Read permission should not be required for delete - $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); } try { - $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); - $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) { + $dbForProject->deleteDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $documentId + ); }); } catch (NotFoundException $e) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3820,10 +3795,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3852,6 +3826,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->noContent(); diff --git a/app/init.php b/app/init.php index d942e056db..cf41bc8049 100644 --- a/app/init.php +++ b/app/init.php @@ -233,7 +233,9 @@ const API_KEY_DYNAMIC = 'dynamic'; const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From c7841bb36ef0b503a6d1001ecfc78ca8ea360e44 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:59:59 +0200 Subject: [PATCH 18/48] database crud usage addition --- app/controllers/api/databases.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index ea17065d45..0ab84bf0fc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3121,7 +3121,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3285,9 +3284,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('mode') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); From fbbc9312eaf3ce85042efd74eb09f3e708394d7f Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:09:44 +0200 Subject: [PATCH 19/48] database crud usage addition --- app/controllers/api/databases.php | 61 +++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0ab84bf0fc..1249ccb2fa 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3036,8 +3036,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3049,6 +3051,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3093,8 +3099,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); @@ -3170,8 +3176,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { if ($document->isEmpty()) { return false; } @@ -3188,6 +3196,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3198,6 +3210,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); + // todo: Use local cache for this getDocument $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); foreach ($relations as $index => $doc) { @@ -3250,8 +3263,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->dynamic(new Document([ @@ -3311,8 +3324,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { if ($document->isEmpty()) { return; } @@ -3328,6 +3343,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3351,8 +3370,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -3649,8 +3668,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3662,6 +3683,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3685,8 +3710,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -3771,8 +3796,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3784,6 +3811,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3823,8 +3854,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->noContent(); From c0d48efe18b7ea40f654a7303c7f2192a4e5bf60 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:29:55 +0200 Subject: [PATCH 20/48] composer --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 1249ccb2fa..a0fea5e293 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); From e7e095907cdf58a2a5627109f9b164e60a34e021 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:44:56 +0200 Subject: [PATCH 21/48] composer --- app/controllers/api/databases.php | 4 ++-- app/init.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a0fea5e293..1249ccb2fa 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); diff --git a/app/init.php b/app/init.php index cf41bc8049..2f83e7c3b2 100644 --- a/app/init.php +++ b/app/init.php @@ -232,9 +232,9 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; -const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; From 504b7276cb3fc0c94967fd177151a629e629b50b Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 11:02:53 +0200 Subject: [PATCH 22/48] composer --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 1249ccb2fa..a0fea5e293 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); From 63cf093020451c8fba1668443e4a5432621d2ebe Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 13:53:57 +0200 Subject: [PATCH 23/48] file transformations usage --- app/controllers/api/storage.php | 15 ++++++++++++++- app/init.php | 10 ++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 7d4361a192..97fb5cc711 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -6,6 +6,7 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Database\Validator\CustomId; @@ -886,7 +887,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal) { + ->inject('queueForUsage') + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, Usage $queueForUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1014,6 +1016,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; + foreach([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $param){ + if(!empty($param)){ + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; + break; + } + } + + $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days ->setContentType($contentType) diff --git a/app/init.php b/app/init.php index 2f83e7c3b2..5e25fa905f 100644 --- a/app/init.php +++ b/app/init.php @@ -232,10 +232,6 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; -const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; -const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; -const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; @@ -263,9 +259,15 @@ const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; +const METRIC_FILES_TRANSFORMATIONS = 'files.transformations'; +const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; const METRIC_FUNCTIONS = 'functions'; From 6b45293f7828f36ee590019a1593c84a6b175747 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 13:55:30 +0200 Subject: [PATCH 24/48] file transformations usage --- app/controllers/api/storage.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 97fb5cc711..8431fb19e3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1016,14 +1016,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - foreach([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $param){ - if(!empty($param)){ - $queueForUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - break; - } + foreach ([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $parameter) { + if (!empty($parameter)) { + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; + break; + } } From bdb93cd3234a7507e365af2f95210715b752fe84 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 2 Jan 2025 19:03:28 +0530 Subject: [PATCH 25/48] Update general.php --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index bb60b01ddd..db9df71341 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -835,7 +835,7 @@ App::error() $adapter = new Sentry($projectId, $key, $host); $logger = new Logger($adapter); - $logger->setSample(0.04); + $logger->setSample(0.01); $publish = true; } else { throw new \Exception('Invalid experimental logging provider'); From c5a49cc5375ab4fdc512fe9207802bd6f4fe18df Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 16:45:21 +0200 Subject: [PATCH 26/48] added operations debug header --- app/controllers/api/databases.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a0fea5e293..c3f74389cc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3077,6 +3077,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + + $response->addHeader('X-Debug-Operations', $operations); $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3098,10 +3105,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setPayload($response->getPayload(), sensitive: $relationships); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3267,6 +3270,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3374,6 +3378,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3714,6 +3719,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( @@ -3858,6 +3864,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 4acc2736e40a21b1c6ecfb9e97b1298086888b25 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 16:46:38 +0200 Subject: [PATCH 27/48] added operations debug header --- app/controllers/api/storage.php | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8431fb19e3..e92b55f140 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1016,16 +1016,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - foreach ([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $parameter) { - if (!empty($parameter)) { - $queueForUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - break; - } - } - + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days From 06a5876f35e86b48b4883f26651b27069d6e51e4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 2 Jan 2025 17:06:22 +0200 Subject: [PATCH 28/48] No throwing Exception --- src/Appwrite/Platform/Workers/Databases.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 9c11dd4090..74c1250505 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -284,8 +284,9 @@ class Databases extends Action $dbForProject->deleteDocument('attributes', $relatedAttribute->getId()); } - throw $e; - + /** + * No throwing Exception + */ } catch (\Throwable $e) { Console::error($e->getMessage()); From bd083ac2805aeb66ee5b11c65cbdccd946edb829 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 2 Jan 2025 17:54:47 +0200 Subject: [PATCH 29/48] database 0.53.29 --- composer.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/composer.lock b/composer.lock index ce5f358619..dbe9e079b5 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.1", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" + "reference": "339ba21476eb184290361542f732ad12c97591ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", + "reference": "339ba21476eb184290361542f732ad12c97591ec", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.1" + "source": "https://github.com/symfony/http-client/tree/v7.2.2" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:50:44+00:00" + "time": "2024-12-30T18:35:15+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.18.3", + "version": "v1.19.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", + "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.65.0", - "illuminate/view": "^10.48.24", - "larastan/larastan": "^2.9.11", - "laravel-zero/framework": "^10.4.0", + "friendsofphp/php-cs-fixer": "^3.66.0", + "illuminate/view": "^10.48.25", + "larastan/larastan": "^2.9.12", + "laravel-zero/framework": "^10.48.25", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-11-26T15:34:00+00:00" + "time": "2024-12-30T16:20:10+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.0", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" + "reference": "87a71856f2f56e4100373e92529eed3171695cfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", + "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", + "reference": "87a71856f2f56e4100373e92529eed3171695cfb", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.0" + "source": "https://github.com/symfony/finder/tree/v7.2.2" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-10-23T06:56:12+00:00" + "time": "2024-12-30T19:00:17+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From e8e39dbd568a202eb7305acc9364d761ae1d2c9b Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 2 Jan 2025 17:57:46 +0200 Subject: [PATCH 30/48] Revert lock --- composer.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/composer.lock b/composer.lock index dbe9e079b5..ce5f358619 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.2", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "339ba21476eb184290361542f732ad12c97591ec" + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", - "reference": "339ba21476eb184290361542f732ad12c97591ec", + "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.2" + "source": "https://github.com/symfony/http-client/tree/v7.2.1" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-30T18:35:15+00:00" + "time": "2024-12-07T08:50:44+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.19.0", + "version": "v1.18.3", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" + "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", - "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", + "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", + "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.66.0", - "illuminate/view": "^10.48.25", - "larastan/larastan": "^2.9.12", - "laravel-zero/framework": "^10.48.25", + "friendsofphp/php-cs-fixer": "^3.65.0", + "illuminate/view": "^10.48.24", + "larastan/larastan": "^2.9.11", + "laravel-zero/framework": "^10.4.0", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-12-30T16:20:10+00:00" + "time": "2024-11-26T15:34:00+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.2", + "version": "v7.2.0", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "87a71856f2f56e4100373e92529eed3171695cfb" + "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", - "reference": "87a71856f2f56e4100373e92529eed3171695cfb", + "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", + "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.2" + "source": "https://github.com/symfony/finder/tree/v7.2.0" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-12-30T19:00:17+00:00" + "time": "2024-10-23T06:56:12+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 1a73847c4798e77923da52a1f231f9fc7bb0cf70 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 11:28:26 +0200 Subject: [PATCH 31/48] added operations debug header --- app/controllers/api/databases.php | 67 ++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c3f74389cc..16258e9972 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,7 +2944,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database) { + $operations = 1; + + $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -2976,10 +2978,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') if ($isList) { $relations = $related; + $operations += count($related); } else { $relations = [$related]; + $operations++; } + $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip( fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) @@ -3026,6 +3031,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ; + try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3036,7 +3046,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3078,8 +3088,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3238,6 +3248,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ; + $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); }, false); @@ -3265,11 +3280,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; - $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, @@ -3586,7 +3596,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { + + $operations = 1; + + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3595,6 +3608,13 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + /** + * Write relationship counts + */ + if (\in_array(\gettype($related), ['array', 'object'])) { + + } + if (empty($related)) { continue; } @@ -3603,8 +3623,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum if ($isList) { $relations = $related; + $operations += count($related); } else { $relations = [$related]; + $operations++; } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); @@ -3654,6 +3676,11 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ; + try { $document = $dbForProject->withRequestTimestamp( $requestTimestamp, @@ -3673,7 +3700,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3715,8 +3742,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->addHeader('X-Debug-Operations', $operations); @@ -3843,6 +3870,15 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + + $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( @@ -3859,11 +3895,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 895e79e7369715be206e01cfdf8fe3eca3376450 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 12:25:56 +0200 Subject: [PATCH 32/48] Operations --- app/controllers/api/databases.php | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 16258e9972..ac41f2d2fe 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,7 +2944,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; + $operations = 1; // For root createDocument $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); @@ -3046,7 +3046,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; + $operations = 0; // Since the Document is sent through post no getDocument used // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3597,7 +3597,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); - $operations = 1; + $operations = 1; // Root updateDocument $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( @@ -3700,7 +3700,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; + $operations = 1; // Since we read original Document // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3811,6 +3811,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; // Read original Document + // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3829,8 +3831,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; - // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); @@ -3877,8 +3877,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - - $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( From 1bc419b6a1dc697cb106b47ca674e4f5b661db48 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Mon, 6 Jan 2025 13:48:01 +0100 Subject: [PATCH 33/48] feat(swoole): allow configuration override of available cpus --- app/http.php | 2 +- app/init.php | 2 +- app/realtime.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/http.php b/app/http.php index 61afce3eae..3a7562ffd1 100644 --- a/app/http.php +++ b/app/http.php @@ -42,7 +42,7 @@ $http = new Server( ); $payloadSize = 12 * (1024 * 1024); // 12MB - adding slight buffer for headers and other data that might be sent with the payload - update later with valid testing -$totalWorkers = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); +$totalWorkers = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $http ->set([ diff --git a/app/init.php b/app/init.php index c6847610c2..7f95ac4c25 100644 --- a/app/init.php +++ b/app/init.php @@ -895,7 +895,7 @@ $register->set('pools', function () { $multiprocessing = System::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; if ($multiprocessing) { - $workerCount = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); + $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); } else { $workerCount = 1; } diff --git a/app/realtime.php b/app/realtime.php index 4f87e4dea1..86f9c85fdd 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -193,7 +193,7 @@ $stats->create(); $containerId = uniqid(); $statsDocument = null; -$workerNumber = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); +$workerNumber = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $adapter = new Adapter\Swoole(port: System::getEnv('PORT', 80)); $adapter From 0814fe49e204221ac86f021f325cc1c0fd92cd25 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 16:50:23 +0200 Subject: [PATCH 34/48] added operations debug header --- app/controllers/api/databases.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 16258e9972..8f59033795 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3031,11 +3031,14 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); + $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-operations-write', $operations); + try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3092,8 +3095,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-read', $operations); $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3253,6 +3255,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-operations-reads', $operations); + $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); }, false); @@ -3280,7 +3284,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } - $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3388,7 +3391,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-reads', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3681,6 +3684,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-operations-writes', $operations); + try { $document = $dbForProject->withRequestTimestamp( $requestTimestamp, @@ -3746,7 +3751,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-reads', $operations); + $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( From f70222d41a532d00bb2db34a767bc64c7f2e96f4 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 17:19:39 +0200 Subject: [PATCH 35/48] added operations debug header --- app/controllers/api/databases.php | 35 ++++++++++--------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 3f8fe7cfdc..fa12087463 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,8 +2944,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; // For root createDocument - + $operations = 1; $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -3031,14 +3030,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); - - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ; - - $response->addHeader('X-Debug-operations-write', $operations); - try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3049,10 +3040,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; // Since the Document is sent through post no getDocument used // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3064,10 +3054,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3091,11 +3077,12 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-operations-read', $operations); + $response->addHeader('X-Debug-Operations', $operations); + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3255,7 +3242,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-operations-reads', $operations); + $response->addHeader('X-Debug-Operations', $operations); $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); @@ -3391,7 +3378,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-operations-reads', $operations); + $response->addHeader('X-Debug-Operations', $operations); + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3600,8 +3588,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); - $operations = 1; // Root updateDocument - + $operations = 1; $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3684,7 +3671,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; - $response->addHeader('X-Debug-operations-writes', $operations); + $response->addHeader('X-Debug-Operations', $operations); try { $document = $dbForProject->withRequestTimestamp( From fc03075ef895d4818029445d6af32fd55cfd4ae0 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 18:01:44 +0200 Subject: [PATCH 36/48] Update counts --- app/controllers/api/databases.php | 28 +++++++++++--------- composer.lock | 44 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index fa12087463..f6de0d7d0f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3163,7 +3163,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); } - $documentId = $cursor->getValue(); $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3178,7 +3177,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { @@ -3186,6 +3185,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') return false; } + $operations++; + $document->removeAttribute('$collection'); $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3198,13 +3199,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { + if (\gettype($related) === 'object'){ $operations++; + } else if(\gettype($related) === 'array'){ + $operations += max(1, count($related)); } if (empty($related)) { continue; } + if (!\is_array($related)) { $relations = [$related]; } else { @@ -3328,7 +3332,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3336,6 +3340,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen return; } + $operations++; + $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3347,13 +3353,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { + if (\gettype($related) === 'object'){ $operations++; + } else if(\gettype($related) === 'array'){ + $operations += max(1, count($related)); } if (empty($related)) { continue; } + if (!\is_array($related)) { $related = [$related]; } @@ -3587,8 +3596,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $operations = 1; + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3598,13 +3607,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - /** - * Write relationship counts - */ - if (\in_array(\gettype($related), ['array', 'object'])) { - - } - if (empty($related)) { continue; } diff --git a/composer.lock b/composer.lock index ce5f358619..dbe9e079b5 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.1", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" + "reference": "339ba21476eb184290361542f732ad12c97591ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", + "reference": "339ba21476eb184290361542f732ad12c97591ec", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.1" + "source": "https://github.com/symfony/http-client/tree/v7.2.2" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:50:44+00:00" + "time": "2024-12-30T18:35:15+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.18.3", + "version": "v1.19.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", + "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.65.0", - "illuminate/view": "^10.48.24", - "larastan/larastan": "^2.9.11", - "laravel-zero/framework": "^10.4.0", + "friendsofphp/php-cs-fixer": "^3.66.0", + "illuminate/view": "^10.48.25", + "larastan/larastan": "^2.9.12", + "laravel-zero/framework": "^10.48.25", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-11-26T15:34:00+00:00" + "time": "2024-12-30T16:20:10+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.0", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" + "reference": "87a71856f2f56e4100373e92529eed3171695cfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", + "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", + "reference": "87a71856f2f56e4100373e92529eed3171695cfb", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.0" + "source": "https://github.com/symfony/finder/tree/v7.2.2" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-10-23T06:56:12+00:00" + "time": "2024-12-30T19:00:17+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From d64ea97ee6ff6f895477cb4b54166e9f6261c0e3 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 18:06:18 +0200 Subject: [PATCH 37/48] added operations debug header --- app/controllers/api/databases.php | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index fa12087463..c65c76297d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3692,10 +3692,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Since we read original Document - // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3707,10 +3705,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3733,13 +3727,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; - - $response->addHeader('X-Debug-operations-reads', $operations); - $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( From 40b407027efb1ddcb453b6570cd033ed004ff717 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 18:51:39 +0200 Subject: [PATCH 38/48] Count changes --- app/controllers/api/databases.php | 49 ++++++++++++------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b9614f5ef9..ca81519834 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,8 +2944,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; + $operations = 0; + $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { + $operations++; + $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -2977,13 +2980,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') if ($isList) { $relations = $related; - $operations += count($related); } else { $relations = [$related]; - $operations++; } - $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip( fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) @@ -3102,8 +3102,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('collection', $collection) ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); - - }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3199,13 +3197,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\gettype($related) === 'object'){ - $operations++; - } else if(\gettype($related) === 'array'){ - $operations += max(1, count($related)); - } - if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } @@ -3353,13 +3349,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\gettype($related) === 'object'){ - $operations++; - } else if(\gettype($related) === 'array'){ - $operations += max(1, count($related)); - } - if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } @@ -3596,9 +3590,12 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $operations = 1; + $operations = 0; $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { + + $operations++; + $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3615,10 +3612,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum if ($isList) { $relations = $related; - $operations += count($related); } else { $relations = [$related]; - $operations++; } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); @@ -3793,8 +3788,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Read original Document - // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3814,7 +3807,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3826,10 +3819,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3855,8 +3844,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $relationships = \array_map( @@ -3875,7 +3862,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-Operations', 1); $response->noContent(); }); From a481e958e345b220a98b252277043cd4de8eac47 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 18:57:35 +0200 Subject: [PATCH 39/48] added operations debug header --- app/controllers/api/databases.php | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b9614f5ef9..5503a78051 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3078,7 +3078,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->addHeader('X-Debug-Operations', $operations); @@ -3102,8 +3102,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('collection', $collection) ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); - - }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3177,9 +3175,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $operations = 0; - // Add $collectionId and $databaseId for all documents + $operations = 1; $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { if ($document->isEmpty()) { return false; @@ -3332,9 +3329,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } - $operations = 0; - // Add $collectionId and $databaseId for all documents + $operations = 1; $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { if ($document->isEmpty()) { return; @@ -3491,6 +3487,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); } } + $response->dynamic(new Document([ 'total' => $audit->countLogsByResource($resource), 'logs' => $output, @@ -3597,7 +3594,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); $operations = 1; - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3793,8 +3789,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Read original Document - // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3814,7 +3808,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3826,10 +3820,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3855,10 +3845,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->addHeader('X-Debug-Operations', $operations); + $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( @@ -3875,7 +3865,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 5c097df56a1710925fbb3bbe0973380147975b21 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 19:07:18 +0200 Subject: [PATCH 40/48] added operations debug header --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 92e61ad1d5..347b4ebbef 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3847,7 +3847,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-Operations', 1); $relationships = \array_map( fn ($document) => $document->getAttribute('key'), From f2cbdbae870babc3c92fca3d10d0c4a255e4ecdf Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 19:32:19 +0200 Subject: [PATCH 41/48] added operations debug header --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index dbe9e079b5..b906af3a81 100644 --- a/composer.lock +++ b/composer.lock @@ -8580,5 +8580,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } From 077313fe2d40be73756c4334c150b248841cbad6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 15:46:35 +1300 Subject: [PATCH 42/48] Update src/Appwrite/Platform/Workers/Databases.php --- src/Appwrite/Platform/Workers/Databases.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 74c1250505..c16fba36c1 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -284,9 +284,6 @@ class Databases extends Action $dbForProject->deleteDocument('attributes', $relatedAttribute->getId()); } - /** - * No throwing Exception - */ } catch (\Throwable $e) { Console::error($e->getMessage()); From 2c04ace02236fb01c2aed9229d5da1ecab63e813 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 18:38:24 +1300 Subject: [PATCH 43/48] Add connection for project --- app/init.php | 31 +++++++++++++++++++++++-------- app/worker.php | 39 +++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/app/init.php b/app/init.php index b4e60d7924..51d9c38239 100644 --- a/app/init.php +++ b/app/init.php @@ -74,6 +74,7 @@ use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Logger; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; @@ -1412,7 +1413,26 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { +App::setResource('connectionForProject', function (Group $pools, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $pools + ->get('console') + ->pop(); + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + return $pools + ->get($dsn->getHost()) + ->pop(); +}, ['pools', 'project']); + +App::setResource('dbForProject', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -1424,12 +1444,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $database ->setMetadata('host', \gethostname()) @@ -1452,7 +1467,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform } return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache', 'project']); App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $dbAdapter = $pools diff --git a/app/worker.php b/app/worker.php index 6eb1363e9b..845af12efc 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,6 +16,7 @@ use Appwrite\Event\Migration; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; +use Swoole\Database\DetectsLostConnections; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -25,11 +26,13 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Platform\Service; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Queue\Connection; use Utopia\Queue\Message; @@ -66,13 +69,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); -Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { +Server::setResource('connectionForProject', function (Group $pools, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; + return $pools + ->get('console') + ->pop(); } - $pools = $register->get('pools'); - try { $dsn = new DSN($project->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -80,12 +83,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $adapter = $pools + return $pools ->get($dsn->getHost()) - ->pop() - ->getResource(); + ->pop(); +}, ['pools', 'project']); - $database = new Database($adapter, $cache); +Server::setResource('dbForProject', function (PoolConnection $connectionForProject, Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + $database = new Database($connectionForProject->getResource(), $cache); try { $dsn = new DSN($project->getAttribute('database')); @@ -109,12 +117,12 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, } return $database; -}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); +}, ['connectionForProject', 'cache', 'register', 'message', 'project', 'dbForPlatform']); -Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { +Server::setResource('getProjectDB', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { + return function (Document $project) use ($pools, $connectionForProject, $dbForPlatform, $cache, &$databases): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -146,12 +154,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $databases[$dsn->getHost()] = $database; @@ -171,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; }; -}, ['pools', 'dbForPlatform', 'cache']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache']); Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); From 71b892132bc4ef7d57ee813bd0d1a6cc307dc3d7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 18:39:06 +1300 Subject: [PATCH 44/48] Mark connections as unhealthy on lost connection error from DB --- app/controllers/general.php | 14 +++++++++++++- app/worker.php | 11 ++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index db9df71341..90cb3293e6 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -21,6 +21,7 @@ use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Utopia\App; use Utopia\CLI\Console; @@ -28,6 +29,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -38,6 +40,7 @@ use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Logger\Logger; +use Utopia\Pools\Connection; use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\Validator\Text; @@ -746,7 +749,16 @@ App::error() ->inject('logger') ->inject('log') ->inject('queueForUsage') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { + ->inject('connectionForProject') + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage, Connection $connectionForProject) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); diff --git a/app/worker.php b/app/worker.php index 845af12efc..62882b32b1 100644 --- a/app/worker.php +++ b/app/worker.php @@ -413,7 +413,16 @@ $worker ->inject('log') ->inject('pools') ->inject('project') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project) use ($queueName) { + ->inject('connectionForProject') + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, PoolConnection $connectionForProject) use ($queueName) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy, it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $pools->reclaim(); $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); From 38ee612164c3be3206fd1eaf6288cc37eadede94 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 18:39:26 +1300 Subject: [PATCH 45/48] Update pools --- composer.json | 2 +- composer.lock | 49 +++++++++++++++++++++++++++++-------------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/composer.json b/composer.json index e44dd0bffc..ebbcad4673 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.5.*", + "utopia-php/pools": "dev-feat-connection-health as 0.5.0", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index b906af3a81..90d2d3fe71 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6b136b5490c0d5d331eac0d70bb3e198", + "content-hash": "59c7d90abfa068a55426b786a244d985", "packages": [ { "name": "adhocore/jwt", @@ -3929,16 +3929,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.13", + "version": "0.6.14", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "68d9b0a9477755afcda607e7e8109785cae17a13" + "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13", - "reference": "68d9b0a9477755afcda607e7e8109785cae17a13", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", + "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", "shasum": "" }, "require": { @@ -3979,9 +3979,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.13" + "source": "https://github.com/utopia-php/migration/tree/0.6.14" }, - "time": "2024-11-26T13:57:53+00:00" + "time": "2025-01-08T01:07:25+00:00" }, { "name": "utopia-php/mongo", @@ -4145,25 +4145,25 @@ }, { "name": "utopia-php/pools", - "version": "0.5.0", + "version": "dev-feat-connection-health", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817" + "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/b1d35f023b3ccf0d171fad88cc833cdace0e1e67", + "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67", "shasum": "" }, "require": { - "php": ">=8.0" + "php": ">=8.3" }, "require-dev": { - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.3" + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "11.*" }, "type": "library", "autoload": { @@ -4190,9 +4190,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.5.0" + "source": "https://github.com/utopia-php/pools/tree/feat-connection-health" }, - "time": "2024-04-19T11:11:54+00:00" + "time": "2025-01-08T03:56:11+00:00" }, { "name": "utopia-php/preloader", @@ -8554,9 +8554,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/pools", + "version": "dev-feat-connection-health", + "alias": "0.5.0", + "alias_normalized": "0.5.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/pools": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8580,5 +8589,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } From 39f34135f0780babc4ee69f9bfe0ef7754ccc36a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 20:43:45 +1300 Subject: [PATCH 46/48] Update pools --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 90d2d3fe71..5aa40556e2 100644 --- a/composer.lock +++ b/composer.lock @@ -4149,12 +4149,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67" + "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/b1d35f023b3ccf0d171fad88cc833cdace0e1e67", - "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/3b1913e7e9cb01acac26215abb37eb767c0112ee", + "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee", "shasum": "" }, "require": { @@ -4192,7 +4192,7 @@ "issues": "https://github.com/utopia-php/pools/issues", "source": "https://github.com/utopia-php/pools/tree/feat-connection-health" }, - "time": "2025-01-08T03:56:11+00:00" + "time": "2025-01-08T07:40:13+00:00" }, { "name": "utopia-php/preloader", From 7f6a43cf7b3bde76d4f4ce47dddbed74a7608a03 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 21:00:02 +1300 Subject: [PATCH 47/48] Update to release --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index ebbcad4673..2612403c1a 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "dev-feat-connection-health as 0.5.0", + "utopia-php/pools": "0.6.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 5aa40556e2..045ef31ffa 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "59c7d90abfa068a55426b786a244d985", + "content-hash": "7b5b5926b452186543903eb539f59c2d", "packages": [ { "name": "adhocore/jwt", @@ -4145,16 +4145,16 @@ }, { "name": "utopia-php/pools", - "version": "dev-feat-connection-health", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee" + "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/3b1913e7e9cb01acac26215abb37eb767c0112ee", - "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/59414ab7b57728edfde6d5ccc5a2583b7967ac18", + "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18", "shasum": "" }, "require": { @@ -4190,9 +4190,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/feat-connection-health" + "source": "https://github.com/utopia-php/pools/tree/0.6.0" }, - "time": "2025-01-08T07:40:13+00:00" + "time": "2025-01-08T07:58:42+00:00" }, { "name": "utopia-php/preloader", @@ -8554,18 +8554,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/pools", - "version": "dev-feat-connection-health", - "alias": "0.5.0", - "alias_normalized": "0.5.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/pools": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 941c9f19055928e4eb7b3fc5761b7602df98a99a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 21:15:33 +1300 Subject: [PATCH 48/48] Add connection health check to http error handler --- app/http.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/http.php b/app/http.php index 3a7562ffd1..b988c7d8ac 100644 --- a/app/http.php +++ b/app/http.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Swoole\Constant; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; @@ -17,6 +18,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -346,6 +348,15 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $app->run($request, $response); } catch (\Throwable $th) { + if ( + ($th instanceof PDOException || $th instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($th) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject = $app->getResource('connectionForProject'); + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $logger = $app->getResource("logger");