From 303f7d484589672e8b55d679d7675b8905456f58 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 24 Jan 2025 21:34:00 +0530 Subject: [PATCH 1/9] feat: add fast2SMS adapter --- .env | 4 +--- composer.json | 2 +- composer.lock | 14 +++++++------- src/Appwrite/Platform/Workers/Messaging.php | 18 ++++++++++++++++-- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.env b/.env index 8f7d7996e7..f237a4bc22 100644 --- a/.env +++ b/.env @@ -63,9 +63,7 @@ _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= -_APP_SMS_PROVIDER=sms://username:password@mock -_APP_SMS_FROM=+123456789 -_APP_SMS_PROJECTS_DENY_LIST= + _APP_STORAGE_LIMIT=30000000 _APP_STORAGE_PREVIEW_LIMIT=20000000 _APP_FUNCTIONS_SIZE_LIMIT=30000000 diff --git a/composer.json b/composer.json index e024cff1c6..e7cc2e5fab 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/image": "0.7.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", - "utopia-php/messaging": "0.13.*", + "utopia-php/messaging": "0.14.*", "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.1", diff --git a/composer.lock b/composer.lock index 02c3b65284..f8a8fbefc3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8927ec7d3cfa460ce223e4c13cf61ada", + "content-hash": "b688586471d8fc6cc39ba144044795f9", "packages": [ { "name": "adhocore/jwt", @@ -3878,16 +3878,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.13.0", + "version": "0.14.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "0e3e57351fe4fe875ef3ab9a01a7fff5f022de90" + "reference": "ca87e106c94eecc81f344cddc8582cc1ff2dc85d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/0e3e57351fe4fe875ef3ab9a01a7fff5f022de90", - "reference": "0e3e57351fe4fe875ef3ab9a01a7fff5f022de90", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/ca87e106c94eecc81f344cddc8582cc1ff2dc85d", + "reference": "ca87e106c94eecc81f344cddc8582cc1ff2dc85d", "shasum": "" }, "require": { @@ -3923,9 +3923,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.13.0" + "source": "https://github.com/utopia-php/messaging/tree/0.14.0" }, - "time": "2024-12-05T08:36:07+00:00" + "time": "2025-01-23T17:57:02+00:00" }, { "name": "utopia-php/migration", diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7837957c5d..d4dc36e9e2 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -22,6 +22,8 @@ use Utopia\Messaging\Adapter\Push\APNS; use Utopia\Messaging\Adapter\Push as PushAdapter; use Utopia\Messaging\Adapter\Push\FCM; use Utopia\Messaging\Adapter\SMS as SMSAdapter; +use Utopia\Messaging\Adapter\SMS\Fast2SMS; +use Utopia\Messaging\Adapter\SMS\GEOSMS; use Utopia\Messaging\Adapter\SMS\Mock; use Utopia\Messaging\Adapter\SMS\Msg91; use Utopia\Messaging\Adapter\SMS\Telesign; @@ -439,6 +441,12 @@ class Messaging extends Action 'apiKey' => $user, 'apiSecret' => $password ], + 'fast2sms' => [ + 'senderId' => $user, + 'apiKey' => $password, + 'messageId' => $smsDSN->getParam('messageId'), + 'useDLT' => $smsDSN->getParam('useDLT'), + ], default => null }, 'options' => match ($host) { @@ -465,7 +473,7 @@ class Messaging extends Action $data = $this->buildSmsMessage($message, $provider); try { - $adapter->send($data); + $result = $adapter->send($data); } catch (\Throwable $th) { throw new \Exception('Failed sending to targets with error: ' . $th->getMessage()); } @@ -474,7 +482,6 @@ class Messaging extends Action } - private function getSmsAdapter(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); @@ -504,6 +511,13 @@ class Messaging extends Action $credentials['apiKey'] ?? '', $credentials['apiSecret'] ?? '' ), + 'fast2sms' => new Fast2SMS( + $credentials['apiKey'] ?? '', + $credentials['senderId'] ?? '', + $credentials['messageId'] ?? '', + [], + $credentials['useDLT'] ?? true + ), default => null }; } From bbe52db4ff2b7dad7672fea9203149423f92b43f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 28 Jan 2025 11:29:15 +0530 Subject: [PATCH 2/9] feat: add geosms adapter --- .env | 4 +- composer.json | 2 +- composer.lock | 107 +++------- src/Appwrite/Platform/Workers/Messaging.php | 223 ++++++++++++-------- 4 files changed, 171 insertions(+), 165 deletions(-) diff --git a/.env b/.env index f237a4bc22..8f7d7996e7 100644 --- a/.env +++ b/.env @@ -63,7 +63,9 @@ _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= - +_APP_SMS_PROVIDER=sms://username:password@mock +_APP_SMS_FROM=+123456789 +_APP_SMS_PROJECTS_DENY_LIST= _APP_STORAGE_LIMIT=30000000 _APP_STORAGE_PREVIEW_LIMIT=20000000 _APP_FUNCTIONS_SIZE_LIMIT=30000000 diff --git a/composer.json b/composer.json index e7cc2e5fab..18318654d5 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/image": "0.7.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", - "utopia-php/messaging": "0.14.*", + "utopia-php/messaging": "dev-update-fast2sms-adapter", "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.1", diff --git a/composer.lock b/composer.lock index f8a8fbefc3..d991cab3bd 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b688586471d8fc6cc39ba144044795f9", + "content-hash": "232f9dd47974029f8dcd24ad4cbe6ae9", "packages": [ { "name": "adhocore/jwt", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.0", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "351a30baa79699de3de3a814c8ccc7b52ccdfb1d" + "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/351a30baa79699de3de3a814c8ccc7b52ccdfb1d", - "reference": "351a30baa79699de3de3a814c8ccc7b52ccdfb1d", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/74b1a03263be8c5acb578f41da054b4bac3af4a0", + "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0", "shasum": "" }, "require": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-08T23:50:34+00:00" + "time": "2025-01-20T23:35:16+00:00" }, { "name": "open-telemetry/context", @@ -1493,16 +1493,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.0", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "9a1c3b866239dbff291e5cc555bb7793eab08127" + "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/9a1c3b866239dbff291e5cc555bb7793eab08127", - "reference": "9a1c3b866239dbff291e5cc555bb7793eab08127", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", + "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", "shasum": "" }, "require": { @@ -1579,7 +1579,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-08T23:50:34+00:00" + "time": "2025-01-09T23:17:14+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3878,16 +3878,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.14.0", + "version": "dev-update-fast2sms-adapter", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "ca87e106c94eecc81f344cddc8582cc1ff2dc85d" + "reference": "01a20625dbe1d552e09084f81c8e3b9ccc25bae5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/ca87e106c94eecc81f344cddc8582cc1ff2dc85d", - "reference": "ca87e106c94eecc81f344cddc8582cc1ff2dc85d", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/01a20625dbe1d552e09084f81c8e3b9ccc25bae5", + "reference": "01a20625dbe1d552e09084f81c8e3b9ccc25bae5", "shasum": "" }, "require": { @@ -3923,9 +3923,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.14.0" + "source": "https://github.com/utopia-php/messaging/tree/update-fast2sms-adapter" }, - "time": "2025-01-23T17:57:02+00:00" + "time": "2025-01-28T04:58:00+00:00" }, { "name": "utopia-php/migration", @@ -5601,70 +5601,18 @@ }, "time": "2023-10-30T13:38:26+00:00" }, - { - "name": "phpbench/dom", - "version": "0.3.3", - "source": { - "type": "git", - "url": "https://github.com/phpbench/dom.git", - "reference": "786a96db538d0def931f5b19225233ec42ec7a72" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpbench/dom/zipball/786a96db538d0def931f5b19225233ec42ec7a72", - "reference": "786a96db538d0def931f5b19225233ec42ec7a72", - "shasum": "" - }, - "require": { - "ext-dom": "*", - "php": "^7.3||^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.14", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^8.0||^9.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0-dev" - } - }, - "autoload": { - "psr-4": { - "PhpBench\\Dom\\": "lib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Daniel Leech", - "email": "daniel@dantleech.com" - } - ], - "description": "DOM wrapper to simplify working with the PHP DOM implementation", - "support": { - "issues": "https://github.com/phpbench/dom/issues", - "source": "https://github.com/phpbench/dom/tree/0.3.3" - }, - "abandoned": true, - "time": "2023-03-06T23:46:57+00:00" - }, { "name": "phpbench/phpbench", - "version": "1.3.1", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0" + "reference": "4248817222514421cba466bfa7adc7d8932345d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0", - "reference": "a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/4248817222514421cba466bfa7adc7d8932345d4", + "reference": "4248817222514421cba466bfa7adc7d8932345d4", "shasum": "" }, "require": { @@ -5677,7 +5625,6 @@ "ext-tokenizer": "*", "php": "^8.1", "phpbench/container": "^2.2", - "phpbench/dom": "~0.3.3", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", "symfony/console": "^6.1 || ^7.0", @@ -5696,8 +5643,8 @@ "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", - "phpunit/phpunit": "^10.4", - "rector/rector": "^0.18.11 || ^1.0.0", + "phpunit/phpunit": "^10.4 || ^11.0", + "rector/rector": "^1.2", "symfony/error-handler": "^6.1 || ^7.0", "symfony/var-dumper": "^6.1 || ^7.0" }, @@ -5742,7 +5689,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.3.1" + "source": "https://github.com/phpbench/phpbench/tree/1.4.0" }, "funding": [ { @@ -5750,7 +5697,7 @@ "type": "github" } ], - "time": "2024-06-30T11:04:37+00:00" + "time": "2025-01-26T19:54:45+00:00" }, { "name": "phpdocumentor/reflection-common", @@ -8556,7 +8503,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/messaging": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index d4dc36e9e2..7abd6cf857 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -48,6 +48,10 @@ class Messaging extends Action { private ?Local $localDevice = null; + private array $dsns = []; + + private ?SMSAdapter $adapter = null; + public static function getName(): string { return 'messaging'; @@ -58,6 +62,21 @@ class Messaging extends Action */ public function __construct() { + if (empty(System::getEnv('_APP_SMS_PROVIDER')) || empty(System::getEnv('_APP_SMS_FROM'))) { + throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); + } + + $providers = System::getEnv('_APP_SMS_PROVIDER', ''); + + if (!empty($providers)) { + $providers = explode(',', $providers); + foreach ($providers as $provider) { + $this->dsns[] = new DSN($provider); + } + } + + $this->adapter = $this->createInternalSMSAdapter($this->dsns); + $this ->desc('Messaging worker') ->inject('message') @@ -383,102 +402,31 @@ class Messaging extends Action private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { - if (empty(System::getEnv('_APP_SMS_PROVIDER')) || empty(System::getEnv('_APP_SMS_FROM'))) { - throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); - } - if ($project->isEmpty()) { throw new \Exception('Project not set in payload'); } - Console::log('Project: ' . $project->getId()); - + Console::log('Processing project: ' . $project->getId()); $denyList = System::getEnv('_APP_SMS_PROJECTS_DENY_LIST', ''); $denyList = explode(',', $denyList); - if (\in_array($project->getId(), $denyList)) { Console::error('Project is in the deny list. Skipping...'); return; } - $smsDSN = new DSN(System::getEnv('_APP_SMS_PROVIDER')); - $host = $smsDSN->getHost(); - $password = $smsDSN->getPassword(); - $user = $smsDSN->getUser(); - - $log->addTag('type', $host); - - $from = System::getEnv('_APP_SMS_FROM'); - - $provider = new Document([ - '$id' => ID::unique(), - 'provider' => $host, - 'type' => MESSAGE_TYPE_SMS, - 'name' => 'Internal SMS', - 'enabled' => true, - 'credentials' => match ($host) { - 'twilio' => [ - 'accountSid' => $user, - 'authToken' => $password, - // Twilio Messaging Service SIDs always start with MG - // https://www.twilio.com/docs/messaging/services - 'messagingServiceSid' => \str_starts_with($from, 'MG') ? $from : null - ], - 'textmagic' => [ - 'username' => $user, - 'apiKey' => $password - ], - 'telesign' => [ - 'customerId' => $user, - 'apiKey' => $password - ], - 'msg91' => [ - 'senderId' => $user, - 'authKey' => $password, - 'templateId' => $smsDSN->getParam('templateId', $from), - ], - 'vonage' => [ - 'apiKey' => $user, - 'apiSecret' => $password - ], - 'fast2sms' => [ - 'senderId' => $user, - 'apiKey' => $password, - 'messageId' => $smsDSN->getParam('messageId'), - 'useDLT' => $smsDSN->getParam('useDLT'), - ], - default => null - }, - 'options' => match ($host) { - 'twilio' => [ - 'from' => \str_starts_with($from, 'MG') ? null : $from - ], - default => [ - 'from' => $from - ] - } - ]); - - $adapter = $this->getSmsAdapter($provider); - - $batches = \array_chunk( - $recipients, - $adapter->getMaxMessagesPerRequest() + $from = System::getEnv('_APP_SMS_FROM', ''); + $sms = new SMS( + $recipients, + $message->getAttribute('data')['content'], + $from ); - batch(\array_map(function ($batch) use ($message, $provider, $adapter) { - return function () use ($batch, $message, $provider, $adapter) { - $message->setAttribute('to', $batch); - - $data = $this->buildSmsMessage($message, $provider); - - try { - $result = $adapter->send($data); - } catch (\Throwable $th) { - throw new \Exception('Failed sending to targets with error: ' . $th->getMessage()); - } - }; - }, $batches)); + try { + $result = $this->adapter->send($sms); + var_dump($result); + } catch (\Throwable $th) { + throw new \Exception('Failed sending to targets with error: ' . $th->getMessage()); + } } @@ -515,7 +463,6 @@ class Messaging extends Action $credentials['apiKey'] ?? '', $credentials['senderId'] ?? '', $credentials['messageId'] ?? '', - [], $credentials['useDLT'] ?? true ), default => null @@ -734,4 +681,112 @@ class Messaging extends Action return $this->localDevice; } + + private function createInternalSMSAdapter(array $dsns): SMSAdapter + { + if (count($dsns) === 1) { + $provider = $this->createProviderFromDSN($dsns[0]); + $adapter = $this->getSmsAdapter($provider); + return $adapter; + } + + $defaultDSN = null; + $localDSNs = []; + + /** @var DSN $dsn */ + foreach ($dsns as $dsn) { + if ($dsn->getParam('local', '') === 'default') { + $defaultDSN = $dsn; + } else { + $localDSNs[] = $dsn; + } + } + + if ($defaultDSN === null) { + throw new \Exception('No default SMS provider found'); + } + + $defaultProvider = $this->createProviderFromDSN($defaultDSN); + $adapter = $this->getSmsAdapter($defaultProvider); + $geosms = new GEOSMS($adapter); + + /** @var DSN $localDSN */ + foreach ($localDSNs as $localDSN) { + try { + $provider = $this->createProviderFromDSN($localDSN); + $adapter = $this->getSmsAdapter($provider); + } catch (\Exception) { + Console::warning('Unable to create adapter: ' . $localDSN->getHost()); + continue; + } + + $callingCode = $localDSN->getParam('local', ''); + if (empty($callingCode)) { + Console::warning('Unable to register adapter: ' . $localDSN->getHost() . '. Missing `local` parameter.'); + continue; + } + + $geosms->setLocal($callingCode, $adapter); + } + return $geosms; + } + + private function createProviderFromDSN(DSN $dsn): Document + { + $host = $dsn->getHost(); + $password = $dsn->getPassword(); + $user = $dsn->getUser(); + $from = System::getEnv('_APP_SMS_FROM'); + + $provider = new Document([ + '$id' => ID::unique(), + 'provider' => $host, + 'type' => MESSAGE_TYPE_SMS, + 'name' => 'Internal SMS', + 'enabled' => true, + 'credentials' => match ($host) { + 'twilio' => [ + 'accountSid' => $user, + 'authToken' => $password, + // Twilio Messaging Service SIDs always start with MG + // https://www.twilio.com/docs/messaging/services + 'messagingServiceSid' => \str_starts_with($from, 'MG') ? $from : null + ], + 'textmagic' => [ + 'username' => $user, + 'apiKey' => $password + ], + 'telesign' => [ + 'customerId' => $user, + 'apiKey' => $password + ], + 'msg91' => [ + 'senderId' => $user, + 'authKey' => $password, + 'templateId' => $dsn->getParam('templateId', $from), + ], + 'vonage' => [ + 'apiKey' => $user, + 'apiSecret' => $password + ], + 'fast2sms' => [ + 'senderId' => $user, + 'apiKey' => $password, + 'messageId' => $dsn->getParam('messageId'), + 'useDLT' => $dsn->getParam('useDLT'), + ], + default => null + }, + 'options' => match ($host) { + 'twilio' => [ + 'from' => \str_starts_with($from, 'MG') ? null : $from + ], + default => [ + 'from' => $from + ] + } + ]); + + return $provider; + } } From dfbe3220d85438ab5cc5d5144879abcae0860f35 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 28 Jan 2025 11:49:29 +0530 Subject: [PATCH 3/9] chore: linter --- composer.json | 2 +- composer.lock | 18 ++++++++---------- src/Appwrite/Platform/Workers/Messaging.php | 4 ++-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/composer.json b/composer.json index 18318654d5..e7cc2e5fab 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/image": "0.7.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", - "utopia-php/messaging": "dev-update-fast2sms-adapter", + "utopia-php/messaging": "0.14.*", "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.1", diff --git a/composer.lock b/composer.lock index d991cab3bd..537c8bfc80 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "232f9dd47974029f8dcd24ad4cbe6ae9", + "content-hash": "b688586471d8fc6cc39ba144044795f9", "packages": [ { "name": "adhocore/jwt", @@ -3878,16 +3878,16 @@ }, { "name": "utopia-php/messaging", - "version": "dev-update-fast2sms-adapter", + "version": "0.14.1", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "01a20625dbe1d552e09084f81c8e3b9ccc25bae5" + "reference": "4ba356a3aa382802727f7e13e0f0152bcc1fc535" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/01a20625dbe1d552e09084f81c8e3b9ccc25bae5", - "reference": "01a20625dbe1d552e09084f81c8e3b9ccc25bae5", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/4ba356a3aa382802727f7e13e0f0152bcc1fc535", + "reference": "4ba356a3aa382802727f7e13e0f0152bcc1fc535", "shasum": "" }, "require": { @@ -3923,9 +3923,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/update-fast2sms-adapter" + "source": "https://github.com/utopia-php/messaging/tree/0.14.1" }, - "time": "2025-01-28T04:58:00+00:00" + "time": "2025-01-28T06:14:28+00:00" }, { "name": "utopia-php/migration", @@ -8503,9 +8503,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/messaging": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7abd6cf857..0f4297c159 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -416,8 +416,8 @@ class Messaging extends Action $from = System::getEnv('_APP_SMS_FROM', ''); $sms = new SMS( - $recipients, - $message->getAttribute('data')['content'], + $recipients, + $message->getAttribute('data')['content'], $from ); From 6cef099890ab886a4e9686d1d226c4102b01a9ad Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 28 Jan 2025 11:51:11 +0530 Subject: [PATCH 4/9] chore: linter --- src/Appwrite/Platform/Workers/Messaging.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 0f4297c159..43fe9b728b 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -423,7 +423,6 @@ class Messaging extends Action try { $result = $this->adapter->send($sms); - var_dump($result); } catch (\Throwable $th) { throw new \Exception('Failed sending to targets with error: ' . $th->getMessage()); } From 8ceb6e2c25e7a26aaf80ea6059d0f1fb8f4efa38 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 28 Jan 2025 12:23:18 +0530 Subject: [PATCH 5/9] chore: review comments --- src/Appwrite/Platform/Workers/Messaging.php | 35 ++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 43fe9b728b..8f2d7547df 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -62,19 +62,7 @@ class Messaging extends Action */ public function __construct() { - if (empty(System::getEnv('_APP_SMS_PROVIDER')) || empty(System::getEnv('_APP_SMS_FROM'))) { - throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); - } - - $providers = System::getEnv('_APP_SMS_PROVIDER', ''); - - if (!empty($providers)) { - $providers = explode(',', $providers); - foreach ($providers as $provider) { - $this->dsns[] = new DSN($provider); - } - } - + $this->adapter = $this->createInternalSMSAdapter($this->dsns); $this @@ -402,6 +390,11 @@ class Messaging extends Action private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { + if ($this->adapter === null) { + Console::warning('Skipped SMS processing. SMS adapter is not set.'); + return; + } + if ($project->isEmpty()) { throw new \Exception('Project not set in payload'); } @@ -681,8 +674,22 @@ class Messaging extends Action return $this->localDevice; } - private function createInternalSMSAdapter(array $dsns): SMSAdapter + private function createInternalSMSAdapter(array $dsns): ?SMSAdapter { + if (empty(System::getEnv('_APP_SMS_PROVIDER')) || empty(System::getEnv('_APP_SMS_FROM'))) { + Console::warning('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); + return null; + } + + $providers = System::getEnv('_APP_SMS_PROVIDER', ''); + + if (!empty($providers)) { + $providers = explode(',', $providers); + foreach ($providers as $provider) { + $this->dsns[] = new DSN($provider); + } + } + if (count($dsns) === 1) { $provider = $this->createProviderFromDSN($dsns[0]); $adapter = $this->getSmsAdapter($provider); From 5593682c3dae58b75b5a6dee6e242c841e63ff20 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 28 Jan 2025 12:24:16 +0530 Subject: [PATCH 6/9] chore: review comments --- src/Appwrite/Platform/Workers/Messaging.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 8f2d7547df..888e083da1 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -62,7 +62,7 @@ class Messaging extends Action */ public function __construct() { - + $this->adapter = $this->createInternalSMSAdapter($this->dsns); $this From 419f3bee1d429ddbbff46c3b692764cf791ad041 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 28 Jan 2025 13:32:10 +0530 Subject: [PATCH 7/9] chore: fix tests --- src/Appwrite/Platform/Workers/Messaging.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 888e083da1..aee60a2bb5 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -48,8 +48,6 @@ class Messaging extends Action { private ?Local $localDevice = null; - private array $dsns = []; - private ?SMSAdapter $adapter = null; public static function getName(): string @@ -63,7 +61,7 @@ class Messaging extends Action public function __construct() { - $this->adapter = $this->createInternalSMSAdapter($this->dsns); + $this->adapter = $this->createInternalSMSAdapter(); $this ->desc('Messaging worker') @@ -674,7 +672,7 @@ class Messaging extends Action return $this->localDevice; } - private function createInternalSMSAdapter(array $dsns): ?SMSAdapter + private function createInternalSMSAdapter(): ?SMSAdapter { if (empty(System::getEnv('_APP_SMS_PROVIDER')) || empty(System::getEnv('_APP_SMS_FROM'))) { Console::warning('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); @@ -683,10 +681,11 @@ class Messaging extends Action $providers = System::getEnv('_APP_SMS_PROVIDER', ''); + $dsns = []; if (!empty($providers)) { $providers = explode(',', $providers); foreach ($providers as $provider) { - $this->dsns[] = new DSN($provider); + $dsns[] = new DSN($provider); } } From 76680bcbcc48d26662872054be539da4ec6a7ec9 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Jan 2025 22:29:24 +1300 Subject: [PATCH 8/9] Revert "Feat batch usage dump" --- composer.json | 6 +- composer.lock | 82 ++--- src/Appwrite/Platform/Workers/Usage.php | 16 +- src/Appwrite/Platform/Workers/UsageDump.php | 361 ++++++++++---------- tests/e2e/General/UsageTest.php | 194 +++++++++++ 5 files changed, 421 insertions(+), 238 deletions(-) diff --git a/composer.json b/composer.json index e024cff1c6..9674bce2fd 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.48.*", + "utopia-php/abuse": "0.47.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.48.*", + "utopia-php/audit": "0.47.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.57.*", + "utopia-php/database": "0.56.4", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 02c3b65284..cd14e512a7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8927ec7d3cfa460ce223e4c13cf61ada", + "content-hash": "3cd37ea04612e04b9e76eb51c5da41dd", "packages": [ { "name": "adhocore/jwt", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.48.0", + "version": "0.47.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "8387c65cc7148af58adbbede06eedc1a7b568e57" + "reference": "2b52bb362234d4072b647ed57db1b3be030f57c2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/8387c65cc7148af58adbbede06eedc1a7b568e57", - "reference": "8387c65cc7148af58adbbede06eedc1a7b568e57", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/2b52bb362234d4072b647ed57db1b3be030f57c2", + "reference": "2b52bb362234d4072b647ed57db1b3be030f57c2", "shasum": "" }, "require": { @@ -3153,13 +3153,13 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.57.*" + "utopia-php/database": "0.56.*" }, "require-dev": { - "laravel/pint": "1.*", - "phpbench/phpbench": "1.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "9.*" + "laravel/pint": "1.5.*", + "phpbench/phpbench": "^1.2", + "phpstan/phpstan": "^1.9", + "phpunit/phpunit": "^9.4" }, "type": "library", "autoload": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.48.0" + "source": "https://github.com/utopia-php/abuse/tree/0.47.0" }, - "time": "2025-01-23T04:40:14+00:00" + "time": "2025-01-15T02:41:02+00:00" }, { "name": "utopia-php/analytics", @@ -3233,26 +3233,26 @@ }, { "name": "utopia-php/audit", - "version": "0.48.0", + "version": "0.47.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "6aab185fce3ba7878b0f26cc8b4eefa1663fb395" + "reference": "1ebd5784ba68645073426f2f04a67726a1bde4d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/6aab185fce3ba7878b0f26cc8b4eefa1663fb395", - "reference": "6aab185fce3ba7878b0f26cc8b4eefa1663fb395", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/1ebd5784ba68645073426f2f04a67726a1bde4d7", + "reference": "1ebd5784ba68645073426f2f04a67726a1bde4d7", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.57.*" + "utopia-php/database": "0.56.*" }, "require-dev": { - "laravel/pint": "1.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "9.*" + "laravel/pint": "1.5.*", + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.3" }, "type": "library", "autoload": { @@ -3274,9 +3274,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.48.0" + "source": "https://github.com/utopia-php/audit/tree/0.47.0" }, - "time": "2025-01-23T04:40:07+00:00" + "time": "2025-01-15T02:40:53+00:00" }, { "name": "utopia-php/cache", @@ -3476,16 +3476,16 @@ }, { "name": "utopia-php/database", - "version": "0.57.2", + "version": "0.56.4", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "bd6f080dd9f4210349a6a862fa6da65a4d9d6339" + "reference": "240478a60797124a885ceac40046fe47c22415b7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/bd6f080dd9f4210349a6a862fa6da65a4d9d6339", - "reference": "bd6f080dd9f4210349a6a862fa6da65a4d9d6339", + "url": "https://api.github.com/repos/utopia-php/database/zipball/240478a60797124a885ceac40046fe47c22415b7", + "reference": "240478a60797124a885ceac40046fe47c22415b7", "shasum": "" }, "require": { @@ -3526,9 +3526,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.57.2" + "source": "https://github.com/utopia-php/database/tree/0.56.4" }, - "time": "2025-01-23T05:19:02+00:00" + "time": "2025-01-20T09:22:08+00:00" }, { "name": "utopia-php/domains", @@ -3929,35 +3929,35 @@ }, { "name": "utopia-php/migration", - "version": "0.6.16", + "version": "0.6.15", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "a1da9b75a0e406ea8caca0d61b57a4d206ea0715" + "reference": "e849ec3e7ad38f5f5273ebb0132b112639cdf01c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/a1da9b75a0e406ea8caca0d61b57a4d206ea0715", - "reference": "a1da9b75a0e406ea8caca0d61b57a4d206ea0715", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/e849ec3e7ad38f5f5273ebb0132b112639cdf01c", + "reference": "e849ec3e7ad38f5f5273ebb0132b112639cdf01c", "shasum": "" }, "require": { - "appwrite/appwrite": "11.*", + "appwrite/appwrite": "11.1.*", "ext-curl": "*", "ext-openssl": "*", - "php": ">=8.3", - "utopia-php/database": "0.57.*", + "php": "8.3.*", + "utopia-php/database": "0.56.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" }, "require-dev": { "ext-pdo": "*", - "laravel/pint": "1.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "11.*", + "laravel/pint": "1.17.*", + "phpstan/phpstan": "1.11.*", + "phpunit/phpunit": "11.2.*", "utopia-php/cli": "0.16.*", - "vlucas/phpdotenv": "5.*" + "vlucas/phpdotenv": "5.6.*" }, "type": "library", "autoload": { @@ -3979,9 +3979,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.16" + "source": "https://github.com/utopia-php/migration/tree/0.6.15" }, - "time": "2025-01-23T04:34:02+00:00" + "time": "2025-01-15T04:55:08+00:00" }, { "name": "utopia-php/mongo", diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3f7428d0dd..3687eeab67 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -30,13 +30,16 @@ class Usage extends Action */ public function __construct() { + $this - ->desc('Usage worker') - ->inject('message') - ->inject('project') - ->inject('getProjectDB') - ->inject('queueForUsageDump') - ->callback([$this, 'action']); + ->desc('Usage worker') + ->inject('message') + ->inject('project') + ->inject('getProjectDB') + ->inject('queueForUsageDump') + ->callback(function (Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $project, $getProjectDB, $queueForUsageDump); + }); $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $this->lastTriggeredTime = time(); @@ -58,6 +61,7 @@ class Usage extends Action throw new Exception('Missing payload'); } + if (empty($project->getAttribute('database'))) { var_dump($payload); return; diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index bb1d605442..2f1d13f29a 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -7,7 +7,7 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception\NotFound; +use Utopia\Database\Exception\Duplicate; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; @@ -38,7 +38,9 @@ class UsageDump extends Action $this ->inject('message') ->inject('getProjectDB') - ->callback([$this, 'action']); + ->callback(function (Message $message, callable $getProjectDB) { + $this->action($message, $getProjectDB); + }); } /** @@ -55,247 +57,230 @@ class UsageDump extends Action throw new Exception('Missing payload'); } - try { - foreach ($payload['stats'] ?? [] as $stats) { - $project = new Document($stats['project'] ?? []); - $numberOfKeys = !empty($stats['keys']) ? \count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? 'NONE'; - if ($numberOfKeys === 0) { - continue; - } + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + + /** + * End temp bug fallback + */ + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + $receivedAt = $stats['receivedAt'] ?? 'NONE'; + if ($numberOfKeys === 0) { + continue; + } + + console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + + try { $dbForProject = $getProjectDB($project); - $projectDocuments = []; - $databaseCache = []; - $collectionSizeCache = []; - - Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys . ' Started'); - $start = \microtime(true); - foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; } + if (str_contains($key, METRIC_DATABASES_STORAGE)) { + try { + $this->handleDatabaseStorage($key, $dbForProject); + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); + } + continue; + } + foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : \date($format, \time()); + $time = 'inf' === $period ? null : date($format, time()); $id = \md5("{$time}_{$period}_{$key}"); - if (\str_contains($key, METRIC_DATABASES_STORAGE)) { - $this->handleDatabaseStorage( - $id, - $key, - $time, - $period, - $dbForProject, - $projectDocuments, - $databaseCache, - $collectionSizeCache - ); - continue; + try { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + $dbForProject->decreaseDocumentAttribute( + 'stats', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats', + $id, + 'value', + $value + ); + } } - - $projectDocuments[] = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); } } - - $dbForProject->createOrUpdateDocumentsWithIncrease( - collection: 'stats', - attribute: 'value', - documents: $projectDocuments - ); - - $end = \microtime(true); - Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s'); + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); } - } catch (\Exception $e) { - Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage()); } } - private function handleDatabaseStorage( - string $id, - string $key, - ?string $time, - string $period, - Database $dbForProject, - array &$projectDocuments, - array &$databaseCache, - array &$collectionSizeCache, - ): void { - $data = \explode('.', $key); - $value = 0; - $previousValue = 0; + private function handleDatabaseStorage(string $key, Database $dbForProject): void + { + $data = explode('.', $key); + $start = microtime(true); - try { - $previousValue = $dbForProject - ->getDocument('stats', $id) - ->getAttribute('value', 0); - } catch (\Exception) { - // No previous value - } + $updateMetric = function (Database $dbForProject, int $value, string $key, string $period, string|null $time) { + $id = \md5("{$time}_{$period}_{$key}"); - switch (\count($data)) { - case METRIC_COLLECTION_LEVEL_STORAGE: - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; - $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}"; + try { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + $dbForProject->decreaseDocumentAttribute( + 'stats', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats', + $id, + 'value', + $value + ); + } + } + }; + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + $value = 0; + $previousValue = 0; + try { + $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); + } catch (\Exception $e) { + // No previous value + } + + switch (count($data)) { + // Collection Level + case METRIC_COLLECTION_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; + $collectionInternalId = $data[1]; - if (!isset($collectionSizeCache[$collectionId])) { try { - $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $collectionSizeCache[$collectionId] = 0; } - } - $value = $collectionSizeCache[$collectionId]; + // Compare with previous value + $diff = $value - $previousValue; - $diff = $value - $previousValue; - if ($diff === 0) { + if ($diff === 0) { + break; + } + + // Update Collection + $updateMetric($dbForProject, $diff, $key, $period, $time); + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); break; - } + // Database Level + case METRIC_DATABASE_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; - $keys = [ - $key, - \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), - METRIC_DATABASES_STORAGE - ]; - - foreach ($keys as $metric) { - $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); - } - break; - - case METRIC_DATABASE_LEVEL_STORAGE: - $databaseInternalId = $data[0]; - $databaseId = "database_{$databaseInternalId}"; - - if (!isset($databaseCache[$databaseId])) { + $collections = []; try { - $databaseCache[$databaseId] = $dbForProject->find($databaseId); + $collections = $dbForProject->find('database_' . $databaseInternalId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Database not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $databaseCache[$databaseId] = []; } - } - foreach ($databaseCache[$databaseId] as $collection) { - $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; - - if (!isset($collectionSizeCache[$collectionId])) { + foreach ($collections as $collection) { try { - $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $collectionSizeCache[$collectionId] = 0; } } - $value += $collectionSizeCache[$collectionId]; - } - $diff = $value - $previousValue; - if ($diff === 0) { + $diff = $value - $previousValue; + + if ($diff === 0) { + break; + } + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); break; - } + // Project Level + case METRIC_PROJECT_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); + // Get all project databases + $databases = $dbForProject->find('database'); - $keys = [ - \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), - METRIC_DATABASES_STORAGE - ]; + // Recalculate all databases + foreach ($databases as $database) { + $collections = $dbForProject->find('database_' . $database->getInternalId()); - foreach ($keys as $metric) { - $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); - } - break; - - case METRIC_PROJECT_LEVEL_STORAGE: - if (!isset($databaseCache['*'])) { - try { - $databaseCache['*'] = $dbForProject->find('databases'); - } catch (\Exception $e) { - if (!$e instanceof NotFound) { - throw $e; - } - $databaseCache['*'] = []; - } - } - - foreach ($databaseCache['*'] as $database) { - $databaseId = "database_{$database->getInternalId()}"; - if (!isset($databaseCache[$databaseId])) { - try { - $databaseCache[$databaseId] = $dbForProject->find($databaseId); - } catch (\Exception $e) { - if (!$e instanceof NotFound) { - throw $e; - } - $databaseCache[$databaseId] = []; - } - } - - foreach ($databaseCache[$databaseId] as $collection) { - $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; - - if (!isset($collectionSizeCache[$collectionId])) { + foreach ($collections as $collection) { try { - $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $collectionSizeCache[$collectionId] = 0; } } - $value += $collectionSizeCache[$collectionId]; } - } - $diff = $value - $previousValue; - if ($diff === 0) { + $diff = $value - $previousValue; + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); break; - } - - $keys = [ - METRIC_DATABASES_STORAGE - ]; - - foreach ($keys as $metric) { - $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); - } - - break; + } } - } - private function createStatsDocument( - string $id, - string $period, - ?string $time, - string $key, - int $diff, - ): Document { - return new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); + $end = microtime(true); + + console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); } } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 3bf727a45e..74ae1c00bc 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -651,6 +651,200 @@ class UsageTest extends Scope ]; } + // /** @depends testDatabaseStoragePrepare */ + // #[Retry(count: 1)] + // public function testDatabaseStorageStatsCreateDocument(array $data): array + // { + // $databaseId = $data['databaseId']; + // $collectionId = $data['collectionId']; + + // $originalProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $originalProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $originalProjectMetrics['body']); + + // $originalProjectMetrics = $originalProjectMetrics['body']; + + // $originalDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $originalDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $originalDatabaseMetrics['body']); + // $originalDatabaseMetrics = $originalDatabaseMetrics['body']; + + // // Create documents + // for ($i = 0; $i < 100; $i++) { + // $response = $this->client->call( + // Client::METHOD_POST, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', + // array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'documentId' => 'unique()', + // 'data' => ['data' => str_repeat('a', 10000)], + // ] + // ); + + // $this->assertEquals(201, $response['headers']['status-code']); + // } + + // sleep(self::WAIT); + + // for ($i = 0; $i < 3; $i++) { + // try { + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); + // $this->assertGreaterThan($originalProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); + + // $newProjectMetrics = $newProjectMetrics['body']; + + // $newDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); + // $this->assertGreaterThan($originalDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); + + // $newDatabaseMetrics = $newDatabaseMetrics['body']; + + // return [ + // 'databaseId' => $databaseId, + // 'collectionId' => $collectionId, + // 'currentProjectMetrics' => $newProjectMetrics, + // 'currentDatabaseMetrics' => $newDatabaseMetrics, + // ]; + // } catch (ExpectationFailedException $e) { + // if ($i === 2) { + // throw $e; + // } + // sleep(self::WAIT); + // continue; + // } + // } + // } + + // /** @depends testDatabaseStorageStatsCreateDocument */ + // #[Retry(count: 1)] + // public function testDatabaseStorageStatsDeleteDocument(array $data): array + // { + // $databaseId = $data['databaseId']; + // $collectionId = $data['collectionId']; + // $currentProjectMetrics = $data['currentProjectMetrics']; + // $currentDatabaseMetrics = $data['currentDatabaseMetrics']; + + // $documents = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', + // array_merge([ + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'queries' => [ + // Query::limit(50)->toString() + // ] + // ] + // ); + + // foreach ($documents['body']['documents'] as $document) { + // $response = $this->client->call( + // Client::METHOD_DELETE, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document['$id'], + // array_merge([ + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()) + // ); + + // $this->assertEquals(204, $response['headers']['status-code']); + // } + + // sleep(self::WAIT); + + // for ($i = 0; $i < 3; $i++) { + // try { + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); + // $this->assertLessThan($currentProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); + + // $newProjectMetrics = $newProjectMetrics['body']; + + // $newDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); + // $this->assertLessThan($currentDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); + + // $newDatabaseMetrics = $newDatabaseMetrics['body']; + + // return [ + // 'databaseId' => $databaseId, + // 'collectionId' => $collectionId, + // 'currentProjectMetrics' => $newProjectMetrics, + // 'currentDatabaseMetrics' => $newDatabaseMetrics, + // ]; + // } catch (ExpectationFailedException $e) { + // if ($i === 2) { + // throw $e; + // } + // sleep(self::WAIT); + // continue; + // } + // } + + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + // } + /** @depends testDatabaseStats */ public function testPrepareFunctionsStats(array $data): array { From 8cf6a4f39335f3eaf65037e1b8b0737be176d6c8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 28 Jan 2025 15:58:16 +0530 Subject: [PATCH 9/9] chore: update deps --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 365541146d..b3c114c81c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3cd37ea04612e04b9e76eb51c5da41dd", + "content-hash": "8a6a8d485f68d2ce144a7ee8bee424ac", "packages": [ { "name": "adhocore/jwt",