From 7a2ee683e3c231eabc15da4c8ea7bb3af21e090a Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 28 Jan 2024 11:28:59 +0200 Subject: [PATCH 01/13] refactor usage poc --- Dockerfile | 3 +- app/controllers/api/users.php | 6 + app/worker.php | 15 +-- bin/worker-usage-dump | 3 + docker-compose.yml | 32 ++++++ src/Appwrite/Event/Event.php | 3 + src/Appwrite/Event/Usage.php | 9 ++ src/Appwrite/Event/UsageDump.php | 47 ++++++++ src/Appwrite/Platform/Services/Workers.php | 4 +- src/Appwrite/Platform/Workers/Usage.php | 69 +++++++++--- src/Appwrite/Platform/Workers/UsageDump.php | 115 ++++++++++++++++++++ src/Appwrite/Platform/Workers/UsageHook.php | 103 ------------------ 12 files changed, 278 insertions(+), 131 deletions(-) create mode 100644 bin/worker-usage-dump create mode 100644 src/Appwrite/Event/UsageDump.php create mode 100644 src/Appwrite/Platform/Workers/UsageDump.php delete mode 100644 src/Appwrite/Platform/Workers/UsageHook.php diff --git a/Dockerfile b/Dockerfile index 324a3a5484..4088d7c8f9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -95,7 +95,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-hamster && \ - chmod +x /usr/local/bin/worker-usage + chmod +x /usr/local/bin/worker-usage && \ + chmod +x /usr/local/bin/worker-usage-dump # Cloud Executabless diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 38d65fba7e..f291213cb9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -9,6 +9,7 @@ use Appwrite\Event\Event; use Appwrite\Network\Validator\Email; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; +use Utopia\CLI\Console; use Utopia\Database\Validator\Queries; use Appwrite\Utopia\Database\Validator\Queries\Users; use Utopia\Database\Validator\Query\Limit; @@ -141,6 +142,8 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); + //Todo debug (to be removed laster @shimon) + //Console::log('@create user=' . time() . '=' . $user->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1191,6 +1194,9 @@ App::delete('/v1/users/:userId') $dbForProject->deleteDocument('users', $userId); + //Todo debug (to be removed laster @shimon) + //Console::log('@delete user=' . $userId . '=' . time() . '=' . $user->getId()); + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($clone); diff --git a/app/worker.php b/app/worker.php index 1b1f4b9f93..4cf0edbae6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -14,6 +14,7 @@ use Appwrite\Event\Mail; use Appwrite\Event\Migration; use Appwrite\Event\Phone; use Appwrite\Event\Usage; +use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\App; @@ -146,6 +147,9 @@ Server::setResource('log', fn() => new Log()); Server::setResource('queueForUsage', function (Connection $queue) { return new Usage($queue); }, ['queue']); +Server::setResource('queueForUsageDump', function (Connection $queue) { + return new UsageDump($queue); +}, ['queue']); Server::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); @@ -299,12 +303,9 @@ $worker Console::error('[Error] Line: ' . $error->getLine()); }); -try { - $workerStart = $worker->getWorkerStart(); -} catch (\Throwable $error) { - $worker->workerStart(); -} finally { - Console::info("Worker $workerName started"); -} +$worker->workerStart() + ->action(function () use ($workerName) { + Console::info("Worker $workerName started"); + }); $worker->start(); diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump new file mode 100644 index 0000000000..43ca87fcb3 --- /dev/null +++ b/bin/worker-usage-dump @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5c645e3bcd..2133de0dce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -676,6 +676,38 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + appwrite-worker-usage-dump: + entrypoint: worker-usage-dump + <<: *x-logging + container_name: appwrite-worker-usage-dump + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + + appwrite-schedule: entrypoint: schedule <<: *x-logging diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index fc12c5b5b3..9f71ef5ebc 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -27,6 +27,9 @@ class Event public const USAGE_QUEUE_NAME = 'v1-usage'; public const USAGE_CLASS_NAME = 'UsageV1'; + public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; + public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; + public const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; public const WEBHOOK_CLASS_NAME = 'WebhooksV1'; diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 398c3319f2..4f53c7df14 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -2,6 +2,7 @@ namespace Appwrite\Event; +use Utopia\CLI\Console; use Utopia\Queue\Client; use Utopia\Queue\Connection; use Utopia\Database\Document; @@ -42,6 +43,14 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { + //Todo debug (to be removed laster @shimon) +// if ($key === 'users') { +// if ($value < 0) { +// console::log('@negative=' . $value); +// } else { +// console::log('@positive=' . $value); +// } +// } $this->metrics[] = [ 'key' => $key, 'value' => $value, diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php new file mode 100644 index 0000000000..8f87908849 --- /dev/null +++ b/src/Appwrite/Event/UsageDump.php @@ -0,0 +1,47 @@ +setQueue(Event::USAGE_DUMP_QUEUE_NAME) + ->setClass(Event::USAGE_DUMP_CLASS_NAME); + } + + /** + * Add Stats. + * + * @param array $stats + * @return self + */ + public function setStats(array $stats): self + { + $this->stats = $stats; + + return $this; + } + + /** + * Sends metrics to the usage worker. + * + * @return string|bool + */ + public function trigger(): string|bool + { + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ + 'stats' => $this->stats, + ]); + } +} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 6573b3124c..22e6dcd56a 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -14,7 +14,7 @@ use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Usage; -use Appwrite\Platform\Workers\UsageHook; +use Appwrite\Platform\Workers\UsageDump; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -33,7 +33,7 @@ class Workers extends Service ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) ->addAction(Hamster::getName(), new Hamster()) - ->addAction(UsageHook::getName(), new UsageHook()) + ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3809d000f7..3527427996 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -4,22 +4,22 @@ namespace Appwrite\Platform\Workers; use Exception; use Utopia\CLI\Console; -use Utopia\Database\Database; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; +use Appwrite\Event\UsageDump; use Utopia\Queue\Message; class Usage extends Action { - protected static array $stats = []; - protected array $periods = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; + private array $stats = []; + private int $lastTriggeredTime = 0; + private int $keys = 0; + private const INFINITY_PERIOD = '_inf_'; + private const KEYS_THRESHOLD = 5; + private const KEYS_SENT_THRESHOLD = 60; + + - protected const INFINITY_PERIOD = '_inf_'; public static function getName(): string { return 'usage'; @@ -35,26 +35,29 @@ class Usage extends Action ->desc('Usage worker') ->inject('message') ->inject('getProjectDB') - ->callback(function (Message $message, callable $getProjectDB) { - $this->action($message, $getProjectDB); + ->inject('queueForUsageDump') + ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $getProjectDB, $queueForUsageDump); }); + + $this->lastTriggeredTime = time(); } /** * @param Message $message * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB): void + public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } - $payload = $message->getPayload() ?? []; $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -69,17 +72,47 @@ class Usage extends Action getProjectDB: $getProjectDB ); } - self::$stats[$projectId]['project'] = $project; + foreach ($payload['metrics'] ?? [] as $metric) { - if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) { - self::$stats[$projectId]['keys'][$metric['key']] = $metric['value']; + if ($metric['key'] === 'users') { + if ($metric['value'] < 0) { + $this->stats[$metric['key']]['negative'] += $metric['value']; + } else { + $this->stats[$metric['key']]['positive'] += $metric['value']; + } + } + } + + $this->stats[$projectId]['project'] = $project; + foreach ($payload['metrics'] ?? [] as $metric) { + $this->keys++; + if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { + $this->stats[$projectId]['keys'][$metric['key']] = $metric['value']; continue; } - self::$stats[$projectId]['keys'][$metric['key']] += $metric['value']; + + $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; + } + + // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) + if ( + $this->keys >= self::KEYS_THRESHOLD || + (time() - $this->lastTriggeredTime > self::KEYS_SENT_THRESHOLD && $this->keys > 0) + ) { + $offset = count($this->stats); + $chunk = array_slice($this->stats, 0, $offset, true); + array_splice($this->stats, 0, $offset); + + $queueForUsageDump + ->setStats($chunk) + ->trigger(); + + //$this->stats = []; + $this->keys = 0; + $this->lastTriggeredTime = time(); } } - /** * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files. * When we remove a parent document we need to deduct his children aggregation from the project scope. diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php new file mode 100644 index 0000000000..e65aec5199 --- /dev/null +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -0,0 +1,115 @@ + 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'usage-dump'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + + $this + ->inject('message') + ->inject('getProjectDB') + ->callback(function (Message $message, callable $getProjectDB) { + $this->action($message, $getProjectDB); + }) + ; + } + + /** + * @param Message $message + * @param callable $getProjectDB + * @return void + * @throws Exception + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, callable $getProjectDB): void + { + + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + $projectInternalId = $project->getInternalId(); + + if ($numberOfKeys === 0) { + continue; + } + + try { + $dbForProject = $getProjectDB($project); + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + try { + $dbForProject->createDocument('stats_v2', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => App::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + var_dump([ + 'id' => $time . '_' . $period . '_' . $key, + 'value' => $value, + ]); + $dbForProject->decreaseDocumentAttribute( + 'stats_v2', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats_v2', + $id, + 'value', + $value + ); + } + } + } + } + } catch (\Exception $e) { + console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); + } + } + } +} diff --git a/src/Appwrite/Platform/Workers/UsageHook.php b/src/Appwrite/Platform/Workers/UsageHook.php deleted file mode 100644 index 4781b1e892..0000000000 --- a/src/Appwrite/Platform/Workers/UsageHook.php +++ /dev/null @@ -1,103 +0,0 @@ -setType(Action::TYPE_WORKER_START) - ->inject('register') - ->inject('getProjectDB') - ->callback(function ($register, callable $getProjectDB) { - $this->action($register, $getProjectDB); - }) - ; - } - - /** - * @param $register - * @param $getProjectDB - * @return void - */ - public function action($register, $getProjectDB): void - { - - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000'); - Timer::tick($interval, function () use ($register, $getProjectDB) { - - $offset = count(self::$stats); - $projects = array_slice(self::$stats, 0, $offset, true); - array_splice(self::$stats, 0, $offset); - foreach ($projects as $data) { - $numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0; - $projectInternalId = $data['project']->getInternalId(); - $database = $data['project']['database'] ?? ''; - - console::warning('Ticker started ' . DateTime::now()); - - if ($numberOfKeys === 0) { - continue; - } - - try { - $dbForProject = $getProjectDB($data['project']); - foreach ($data['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats_v2', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => App::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats_v2', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats_v2', - $id, - 'value', - $value - ); - } - } - } - } - } catch (\Exception $e) { - console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); - } - } - }); - } -} From 36fddd415614062cf651ceb0ff79b8c0aa2695df Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 28 Jan 2024 11:46:13 +0200 Subject: [PATCH 02/13] refactor usage poc --- src/Appwrite/Platform/Workers/UsageDump.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index e65aec5199..d0e383be19 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -86,10 +86,11 @@ class UsageDump extends Action ])); } catch (Duplicate $th) { if ($value < 0) { - var_dump([ - 'id' => $time . '_' . $period . '_' . $key, - 'value' => $value, - ]); + //Todo debug (to be removed laster @shimon) +// var_dump([ +// 'id' => $time . '_' . $period . '_' . $key, +// 'value' => $value, +// ]); $dbForProject->decreaseDocumentAttribute( 'stats_v2', $id, From 44ed6826a9c81ec124b20954cc2f025f27d0da2b Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 28 Jan 2024 15:33:04 +0200 Subject: [PATCH 03/13] refactor usage poc --- app/controllers/api/users.php | 4 ++-- src/Appwrite/Event/Usage.php | 2 +- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index f291213cb9..f3faa7838a 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -142,7 +142,7 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) //Console::log('@create user=' . time() . '=' . $user->getId()); $response @@ -1194,7 +1194,7 @@ App::delete('/v1/users/:userId') $dbForProject->deleteDocument('users', $userId); - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) //Console::log('@delete user=' . $userId . '=' . time() . '=' . $user->getId()); $queueForDeletes diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 4f53c7df14..0774534e36 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -43,7 +43,7 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) // if ($key === 'users') { // if ($value < 0) { // console::log('@negative=' . $value); diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index d0e383be19..5221d58b2c 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -86,7 +86,7 @@ class UsageDump extends Action ])); } catch (Duplicate $th) { if ($value < 0) { - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) // var_dump([ // 'id' => $time . '_' . $period . '_' . $key, // 'value' => $value, From 5518ca165e0f6f5eb82b1af69a8e29ff3e4d38b9 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 Jan 2024 13:24:57 +0200 Subject: [PATCH 04/13] remarks --- app/controllers/api/users.php | 5 ----- src/Appwrite/Platform/Workers/Usage.php | 8 ++------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index f3faa7838a..cc41ea85b0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -142,8 +142,6 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); - //Todo debug (to be removed later @shimon) - //Console::log('@create user=' . time() . '=' . $user->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1194,9 +1192,6 @@ App::delete('/v1/users/:userId') $dbForProject->deleteDocument('users', $userId); - //Todo debug (to be removed later @shimon) - //Console::log('@delete user=' . $userId . '=' . time() . '=' . $user->getId()); - $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($clone); diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3527427996..74b8f4c224 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -99,15 +99,11 @@ class Usage extends Action $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > self::KEYS_SENT_THRESHOLD && $this->keys > 0) ) { - $offset = count($this->stats); - $chunk = array_slice($this->stats, 0, $offset, true); - array_splice($this->stats, 0, $offset); - $queueForUsageDump - ->setStats($chunk) + ->setStats($this->stats) ->trigger(); - //$this->stats = []; + $this->stats = []; $this->keys = 0; $this->lastTriggeredTime = time(); } From 6a6c3445fa646cabfc5ba025eab8405b3d7e3617 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 4 Feb 2024 20:02:01 +0200 Subject: [PATCH 05/13] updates --- app/controllers/api/users.php | 18 +- composer.json | 20 +- composer.lock | 536 ++++++++++++++++++++---- docker-compose.yml | 1 + src/Appwrite/Platform/Workers/Usage.php | 10 - 5 files changed, 484 insertions(+), 101 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index cc41ea85b0..7449ac27c0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -142,7 +142,6 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); - $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($user, Response::MODEL_USER); @@ -1190,15 +1189,16 @@ App::delete('/v1/users/:userId') // clone user object to send to workers $clone = clone $user; - $dbForProject->deleteDocument('users', $userId); + $affected = $dbForProject->deleteDocument('users', $userId); + if (!empty($affected)) { + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($clone); - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($clone); - - $queueForEvents - ->setParam('userId', $user->getId()) - ->setPayload($response->output($clone, Response::MODEL_USER)); + $queueForEvents + ->setParam('userId', $user->getId()) + ->setPayload($response->output($clone, Response::MODEL_USER)); + } $response->noContent(); }); diff --git a/composer.json b/composer.json index 83bcd26468..79021ad522 100644 --- a/composer.json +++ b/composer.json @@ -45,18 +45,18 @@ "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.33.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.35.*", + "utopia-php/audit": "0.38.*", "utopia-php/cache": "0.9.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.45.*", - "utopia-php/domains": "0.3.*", - "utopia-php/dsn": "0.1.*", + "utopia-php/database": "0.45.6", + "utopia-php/domains": "0.5.*", + "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", - "utopia-php/image": "0.5.*", + "utopia-php/image": "0.6.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", - "utopia-php/messaging": "0.2.*", + "utopia-php/messaging": "0.9.*", "utopia-php/migration": "0.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.5.*", @@ -73,8 +73,9 @@ "phpmailer/phpmailer": "6.8.0", "chillerlan/php-qrcode": "4.3.4", "adhocore/jwt": "1.1.2", + "spomky-labs/otphp": "^10.0", "webonyx/graphql-php": "14.11.*", - "league/csv": "9.7.1" + "league/csv": "^9.14" }, "repositories": [ { @@ -88,14 +89,15 @@ "phpunit/phpunit": "9.5.20", "squizlabs/php_codesniffer": "^3.7", "swoole/ide-helper": "5.0.2", - "textalk/websocket": "1.5.7" + "textalk/websocket": "1.5.7", + "utopia-php/fetch": "0.1.*" }, "provide": { "ext-phpiredis": "*" }, "config": { "platform": { - "php": "8.0" + "php": "8.2" } } } diff --git a/composer.lock b/composer.lock index 61a8f297d7..d7a827ccb4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1b3fd261ed93452413b8e6ba77dbab55", + "content-hash": "b70152e4a3c53762b42d9623ea204cf6", "packages": [ { "name": "adhocore/jwt", @@ -197,6 +197,73 @@ ], "time": "2023-11-22T15:36:00+00:00" }, + { + "name": "beberlei/assert", + "version": "v3.3.2", + "source": { + "type": "git", + "url": "https://github.com/beberlei/assert.git", + "reference": "cb70015c04be1baee6f5f5c953703347c0ac1655" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/beberlei/assert/zipball/cb70015c04be1baee6f5f5c953703347c0ac1655", + "reference": "cb70015c04be1baee6f5f5c953703347c0ac1655", + "shasum": "" + }, + "require": { + "ext-ctype": "*", + "ext-json": "*", + "ext-mbstring": "*", + "ext-simplexml": "*", + "php": "^7.0 || ^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "*", + "phpstan/phpstan": "*", + "phpunit/phpunit": ">=6.0.0", + "yoast/phpunit-polyfills": "^0.1.0" + }, + "suggest": { + "ext-intl": "Needed to allow Assertion::count(), Assertion::isCountable(), Assertion::minCount(), and Assertion::maxCount() to operate on ResourceBundles" + }, + "type": "library", + "autoload": { + "files": [ + "lib/Assert/functions.php" + ], + "psr-4": { + "Assert\\": "lib/Assert" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-2-Clause" + ], + "authors": [ + { + "name": "Benjamin Eberlei", + "email": "kontakt@beberlei.de", + "role": "Lead Developer" + }, + { + "name": "Richard Quadling", + "email": "rquadling@gmail.com", + "role": "Collaborator" + } + ], + "description": "Thin assertion library for input validation in business models.", + "keywords": [ + "assert", + "assertion", + "validation" + ], + "support": { + "issues": "https://github.com/beberlei/assert/issues", + "source": "https://github.com/beberlei/assert/tree/v3.3.2" + }, + "time": "2021-12-16T21:41:27+00:00" + }, { "name": "chillerlan/php-qrcode", "version": "4.3.4", @@ -463,34 +530,39 @@ }, { "name": "league/csv", - "version": "9.7.1", + "version": "9.14.0", "source": { "type": "git", "url": "https://github.com/thephpleague/csv.git", - "reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1" + "reference": "34bf0df7340b60824b9449b5c526fcc3325070d5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/csv/zipball/0ec57e8264ec92565974ead0d1724cf1026e10c1", - "reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/34bf0df7340b60824b9449b5c526fcc3325070d5", + "reference": "34bf0df7340b60824b9449b5c526fcc3325070d5", "shasum": "" }, "require": { + "ext-filter": "*", "ext-json": "*", "ext-mbstring": "*", - "php": "^7.3 || ^8.0" + "php": "^8.1.2" }, "require-dev": { - "ext-curl": "*", + "doctrine/collections": "^2.1.4", "ext-dom": "*", - "friendsofphp/php-cs-fixer": "^2.16", - "phpstan/phpstan": "^0.12.0", - "phpstan/phpstan-phpunit": "^0.12.0", - "phpstan/phpstan-strict-rules": "^0.12.0", - "phpunit/phpunit": "^9.5" + "ext-xdebug": "*", + "friendsofphp/php-cs-fixer": "^v3.22.0", + "phpbench/phpbench": "^1.2.15", + "phpstan/phpstan": "^1.10.50", + "phpstan/phpstan-deprecation-rules": "^1.1.4", + "phpstan/phpstan-phpunit": "^1.3.15", + "phpstan/phpstan-strict-rules": "^1.5.2", + "phpunit/phpunit": "^10.5.3", + "symfony/var-dumper": "^6.4.0" }, "suggest": { - "ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes", + "ext-dom": "Required to use the XMLConverter and the HTMLConverter classes", "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" }, "type": "library", @@ -520,7 +592,7 @@ } ], "description": "CSV data manipulation made easy in PHP", - "homepage": "http://csv.thephpleague.com", + "homepage": "https://csv.thephpleague.com", "keywords": [ "convert", "csv", @@ -543,7 +615,7 @@ "type": "github" } ], - "time": "2021-04-17T16:32:08+00:00" + "time": "2023-12-29T07:34:53+00:00" }, { "name": "matomo/device-detector", @@ -733,6 +805,73 @@ ], "time": "2019-09-10T13:16:29+00:00" }, + { + "name": "paragonie/constant_time_encoding", + "version": "v2.6.3", + "source": { + "type": "git", + "url": "https://github.com/paragonie/constant_time_encoding.git", + "reference": "58c3f47f650c94ec05a151692652a868995d2938" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/58c3f47f650c94ec05a151692652a868995d2938", + "reference": "58c3f47f650c94ec05a151692652a868995d2938", + "shasum": "" + }, + "require": { + "php": "^7|^8" + }, + "require-dev": { + "phpunit/phpunit": "^6|^7|^8|^9", + "vimeo/psalm": "^1|^2|^3|^4" + }, + "type": "library", + "autoload": { + "psr-4": { + "ParagonIE\\ConstantTime\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Paragon Initiative Enterprises", + "email": "security@paragonie.com", + "homepage": "https://paragonie.com", + "role": "Maintainer" + }, + { + "name": "Steve 'Sc00bz' Thomas", + "email": "steve@tobtu.com", + "homepage": "https://www.tobtu.com", + "role": "Original Developer" + } + ], + "description": "Constant-time Implementations of RFC 4648 Encoding (Base-64, Base-32, Base-16)", + "keywords": [ + "base16", + "base32", + "base32_decode", + "base32_encode", + "base64", + "base64_decode", + "base64_encode", + "bin2hex", + "encoding", + "hex", + "hex2bin", + "rfc4648" + ], + "support": { + "email": "info@paragonie.com", + "issues": "https://github.com/paragonie/constant_time_encoding/issues", + "source": "https://github.com/paragonie/constant_time_encoding" + }, + "time": "2022-06-14T06:56:20+00:00" + }, { "name": "phpmailer/phpmailer", "version": "v6.8.0", @@ -813,6 +952,81 @@ ], "time": "2023-03-06T14:43:22+00:00" }, + { + "name": "spomky-labs/otphp", + "version": "v10.0.3", + "source": { + "type": "git", + "url": "https://github.com/Spomky-Labs/otphp.git", + "reference": "9784d9f7c790eed26e102d6c78f12c754036c366" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Spomky-Labs/otphp/zipball/9784d9f7c790eed26e102d6c78f12c754036c366", + "reference": "9784d9f7c790eed26e102d6c78f12c754036c366", + "shasum": "" + }, + "require": { + "beberlei/assert": "^3.0", + "ext-mbstring": "*", + "paragonie/constant_time_encoding": "^2.0", + "php": "^7.2|^8.0", + "thecodingmachine/safe": "^0.1.14|^1.0|^2.0" + }, + "require-dev": { + "php-coveralls/php-coveralls": "^2.0", + "phpstan/phpstan": "^0.12", + "phpstan/phpstan-beberlei-assert": "^0.12", + "phpstan/phpstan-deprecation-rules": "^0.12", + "phpstan/phpstan-phpunit": "^0.12", + "phpstan/phpstan-strict-rules": "^0.12", + "phpunit/phpunit": "^8.0", + "thecodingmachine/phpstan-safe-rule": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "v10.0": "10.0.x-dev", + "v9.0": "9.0.x-dev", + "v8.3": "8.3.x-dev" + } + }, + "autoload": { + "psr-4": { + "OTPHP\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Florent Morselli", + "homepage": "https://github.com/Spomky" + }, + { + "name": "All contributors", + "homepage": "https://github.com/Spomky-Labs/otphp/contributors" + } + ], + "description": "A PHP library for generating one time passwords according to RFC 4226 (HOTP Algorithm) and the RFC 6238 (TOTP Algorithm) and compatible with Google Authenticator", + "homepage": "https://github.com/Spomky-Labs/otphp", + "keywords": [ + "FreeOTP", + "RFC 4226", + "RFC 6238", + "google authenticator", + "hotp", + "otp", + "totp" + ], + "support": { + "issues": "https://github.com/Spomky-Labs/otphp/issues", + "source": "https://github.com/Spomky-Labs/otphp/tree/v10.0.3" + }, + "time": "2022-03-17T08:00:35+00:00" + }, { "name": "symfony/polyfill-php80", "version": "v1.28.0", @@ -896,6 +1110,145 @@ ], "time": "2023-01-26T09:26:14+00:00" }, + { + "name": "thecodingmachine/safe", + "version": "v2.5.0", + "source": { + "type": "git", + "url": "https://github.com/thecodingmachine/safe.git", + "reference": "3115ecd6b4391662b4931daac4eba6b07a2ac1f0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thecodingmachine/safe/zipball/3115ecd6b4391662b4931daac4eba6b07a2ac1f0", + "reference": "3115ecd6b4391662b4931daac4eba6b07a2ac1f0", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "phpstan/phpstan": "^1.5", + "phpunit/phpunit": "^9.5", + "squizlabs/php_codesniffer": "^3.2", + "thecodingmachine/phpstan-strict-rules": "^1.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.2.x-dev" + } + }, + "autoload": { + "files": [ + "deprecated/apc.php", + "deprecated/array.php", + "deprecated/datetime.php", + "deprecated/libevent.php", + "deprecated/misc.php", + "deprecated/password.php", + "deprecated/mssql.php", + "deprecated/stats.php", + "deprecated/strings.php", + "lib/special_cases.php", + "deprecated/mysqli.php", + "generated/apache.php", + "generated/apcu.php", + "generated/array.php", + "generated/bzip2.php", + "generated/calendar.php", + "generated/classobj.php", + "generated/com.php", + "generated/cubrid.php", + "generated/curl.php", + "generated/datetime.php", + "generated/dir.php", + "generated/eio.php", + "generated/errorfunc.php", + "generated/exec.php", + "generated/fileinfo.php", + "generated/filesystem.php", + "generated/filter.php", + "generated/fpm.php", + "generated/ftp.php", + "generated/funchand.php", + "generated/gettext.php", + "generated/gmp.php", + "generated/gnupg.php", + "generated/hash.php", + "generated/ibase.php", + "generated/ibmDb2.php", + "generated/iconv.php", + "generated/image.php", + "generated/imap.php", + "generated/info.php", + "generated/inotify.php", + "generated/json.php", + "generated/ldap.php", + "generated/libxml.php", + "generated/lzf.php", + "generated/mailparse.php", + "generated/mbstring.php", + "generated/misc.php", + "generated/mysql.php", + "generated/network.php", + "generated/oci8.php", + "generated/opcache.php", + "generated/openssl.php", + "generated/outcontrol.php", + "generated/pcntl.php", + "generated/pcre.php", + "generated/pgsql.php", + "generated/posix.php", + "generated/ps.php", + "generated/pspell.php", + "generated/readline.php", + "generated/rpminfo.php", + "generated/rrd.php", + "generated/sem.php", + "generated/session.php", + "generated/shmop.php", + "generated/sockets.php", + "generated/sodium.php", + "generated/solr.php", + "generated/spl.php", + "generated/sqlsrv.php", + "generated/ssdeep.php", + "generated/ssh2.php", + "generated/stream.php", + "generated/strings.php", + "generated/swoole.php", + "generated/uodbc.php", + "generated/uopz.php", + "generated/url.php", + "generated/var.php", + "generated/xdiff.php", + "generated/xml.php", + "generated/xmlrpc.php", + "generated/yaml.php", + "generated/yaz.php", + "generated/zip.php", + "generated/zlib.php" + ], + "classmap": [ + "lib/DateTime.php", + "lib/DateTimeImmutable.php", + "lib/Exceptions/", + "deprecated/Exceptions/", + "generated/Exceptions/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHP core functions that throw exceptions instead of returning FALSE on error", + "support": { + "issues": "https://github.com/thecodingmachine/safe/issues", + "source": "https://github.com/thecodingmachine/safe/tree/v2.5.0" + }, + "time": "2023-04-05T11:54:14+00:00" + }, { "name": "utopia-php/abuse", "version": "0.33.0", @@ -1190,16 +1543,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.5", + "version": "0.45.6", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6" + "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/0b66a017f817a910acb83e6aea92bccea9571fe6", - "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd", + "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd", "shasum": "" }, "require": { @@ -1240,22 +1593,22 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.5" + "source": "https://github.com/utopia-php/database/tree/0.45.6" }, - "time": "2024-01-08T17:08:15+00:00" + "time": "2024-02-01T02:33:43+00:00" }, { "name": "utopia-php/domains", - "version": "0.3.2", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb" + "reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/aaa8c9a96c69ccb397997b1f4f2299c66f77eefb", - "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/bf07f60326f8389f378ddf6fcde86217e5cfe18c", + "reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c", "shasum": "" }, "require": { @@ -1300,22 +1653,22 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/0.3.2" + "source": "https://github.com/utopia-php/domains/tree/0.5.0" }, - "time": "2023-07-19T16:39:24+00:00" + "time": "2024-01-03T22:04:27+00:00" }, { "name": "utopia-php/dsn", - "version": "0.1.0", + "version": "0.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/dsn.git", - "reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea" + "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/dsn/zipball/17a5935eab1b89fb4b95600db50a1b6d5faa6cea", - "reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea", + "url": "https://api.github.com/repos/utopia-php/dsn/zipball/c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", + "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", "shasum": "" }, "require": { @@ -1347,22 +1700,22 @@ ], "support": { "issues": "https://github.com/utopia-php/dsn/issues", - "source": "https://github.com/utopia-php/dsn/tree/0.1.0" + "source": "https://github.com/utopia-php/dsn/tree/0.2.0" }, - "time": "2022-10-26T10:06:20+00:00" + "time": "2023-11-02T12:01:43+00:00" }, { "name": "utopia-php/framework", - "version": "0.33.0", + "version": "0.33.2", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "e3ff6b933082d57b48e7c4267bb605c0bf2250fd" + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/e3ff6b933082d57b48e7c4267bb605c0bf2250fd", - "reference": "e3ff6b933082d57b48e7c4267bb605c0bf2250fd", + "url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", "shasum": "" }, "require": { @@ -1392,22 +1745,22 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.0" + "source": "https://github.com/utopia-php/http/tree/0.33.2" }, - "time": "2024-01-08T13:30:27+00:00" + "time": "2024-01-31T10:35:59+00:00" }, { "name": "utopia-php/image", - "version": "0.5.4", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336" + "reference": "88f7209172bdabd81e76ac981c95fac117dc6e08" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/ca5f436f9aa22dedaa6648f24f3687733808e336", - "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336", + "url": "https://api.github.com/repos/utopia-php/image/zipball/88f7209172bdabd81e76ac981c95fac117dc6e08", + "reference": "88f7209172bdabd81e76ac981c95fac117dc6e08", "shasum": "" }, "require": { @@ -1415,6 +1768,8 @@ "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -1428,12 +1783,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple Image manipulation library", "keywords": [ "framework", @@ -1444,9 +1793,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.5.4" + "source": "https://github.com/utopia-php/image/tree/0.6.0" }, - "time": "2022-05-11T12:30:41+00:00" + "time": "2024-01-24T06:59:44+00:00" }, { "name": "utopia-php/locale", @@ -1554,26 +1903,28 @@ }, { "name": "utopia-php/messaging", - "version": "0.2.0", + "version": "0.9.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "2d0f474a106bb1da285f85e105c29b46085d3a43" + "reference": "df54ba51570e886724590edeb03dbd455bb0464d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/2d0f474a106bb1da285f85e105c29b46085d3a43", - "reference": "2d0f474a106bb1da285f85e105c29b46085d3a43", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/df54ba51570e886724590edeb03dbd455bb0464d", + "reference": "df54ba51570e886724590edeb03dbd455bb0464d", "shasum": "" }, "require": { "ext-curl": "*", + "ext-openssl": "*", "php": ">=8.0.0" }, "require-dev": { - "laravel/pint": "^1.2", + "laravel/pint": "1.13.*", "phpmailer/phpmailer": "6.8.*", - "phpunit/phpunit": "9.6.*" + "phpstan/phpstan": "1.10.*", + "phpunit/phpunit": "9.6.10" }, "type": "library", "autoload": { @@ -1596,9 +1947,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.2.0" + "source": "https://github.com/utopia-php/messaging/tree/0.9.0" }, - "time": "2023-09-14T20:48:42+00:00" + "time": "2024-01-31T11:51:27+00:00" }, { "name": "utopia-php/migration", @@ -2420,16 +2771,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.36.0", + "version": "0.36.2", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a" + "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3a10f1f895ed71120442ff71eb6adec3fd6b4e8a", - "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0aa67479d75f0e0cb7b60454031534d7f0abaece", + "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece", "shasum": "" }, "require": { @@ -2465,22 +2816,22 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.36.0" + "source": "https://github.com/appwrite/sdk-generator/tree/0.36.2" }, - "time": "2023-11-20T10:03:06+00:00" + "time": "2024-01-19T01:04:35+00:00" }, { "name": "doctrine/deprecations", - "version": "1.1.2", + "version": "1.1.3", "source": { "type": "git", "url": "https://github.com/doctrine/deprecations.git", - "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931" + "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931", - "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab", + "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab", "shasum": "" }, "require": { @@ -2512,9 +2863,9 @@ "homepage": "https://www.doctrine-project.org/", "support": { "issues": "https://github.com/doctrine/deprecations/issues", - "source": "https://github.com/doctrine/deprecations/tree/1.1.2" + "source": "https://github.com/doctrine/deprecations/tree/1.1.3" }, - "time": "2023-09-27T20:04:15+00:00" + "time": "2024-01-30T19:34:25+00:00" }, { "name": "doctrine/instantiator", @@ -5105,6 +5456,45 @@ } ], "time": "2023-11-21T18:54:41+00:00" + }, + { + "name": "utopia-php/fetch", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.1.0" + }, + "time": "2023-10-10T11:58:32+00:00" } ], "aliases": [], @@ -5131,7 +5521,7 @@ "ext-fileinfo": "*" }, "platform-overrides": { - "php": "8.0" + "php": "8.2" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 2133de0dce..c7080cf2b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev + #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 74b8f4c224..32f636a0f6 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -73,16 +73,6 @@ class Usage extends Action ); } - foreach ($payload['metrics'] ?? [] as $metric) { - if ($metric['key'] === 'users') { - if ($metric['value'] < 0) { - $this->stats[$metric['key']]['negative'] += $metric['value']; - } else { - $this->stats[$metric['key']]['positive'] += $metric['value']; - } - } - } - $this->stats[$projectId]['project'] = $project; foreach ($payload['metrics'] ?? [] as $metric) { $this->keys++; From 93a99b561c68394921470832ac043066b6365d50 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 7 Feb 2024 11:54:18 +0200 Subject: [PATCH 06/13] remove debug lines --- src/Appwrite/Event/Usage.php | 8 -------- src/Appwrite/Platform/Workers/UsageDump.php | 5 ----- 2 files changed, 13 deletions(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 0774534e36..ded276e166 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -43,14 +43,6 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { - //Todo debug (to be removed later @shimon) -// if ($key === 'users') { -// if ($value < 0) { -// console::log('@negative=' . $value); -// } else { -// console::log('@positive=' . $value); -// } -// } $this->metrics[] = [ 'key' => $key, 'value' => $value, diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 5221d58b2c..24fe4ccccd 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -86,11 +86,6 @@ class UsageDump extends Action ])); } catch (Duplicate $th) { if ($value < 0) { - //Todo debug (to be removed later @shimon) -// var_dump([ -// 'id' => $time . '_' . $period . '_' . $key, -// 'value' => $value, -// ]); $dbForProject->decreaseDocumentAttribute( 'stats_v2', $id, From 990698e4239d56cb97997358d0ecead8ee0c0be0 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 09:32:05 +0200 Subject: [PATCH 07/13] composer update --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index e72cc04a15..4d5535b8f8 100644 --- a/composer.lock +++ b/composer.lock @@ -5124,5 +5124,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } From 790cdc297b1663cd79c873e7324e18b705de55dd Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 10:26:45 +0200 Subject: [PATCH 08/13] composer update --- .env | 2 +- app/controllers/api/users.php | 17 ++++++++--------- docker-compose.yml | 1 - src/Appwrite/Platform/Workers/Usage.php | 9 +++++---- src/Appwrite/Platform/Workers/UsageDump.php | 1 + 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.env b/.env index 0247356a25..3efd72e2a2 100644 --- a/.env +++ b/.env @@ -78,7 +78,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=60000 +_APP_USAGE_AGGREGATION_INTERVAL=60 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 7449ac27c0..5ce2263f47 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1189,16 +1189,15 @@ App::delete('/v1/users/:userId') // clone user object to send to workers $clone = clone $user; - $affected = $dbForProject->deleteDocument('users', $userId); - if (!empty($affected)) { - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($clone); + $dbForProject->deleteDocument('users', $userId); - $queueForEvents - ->setParam('userId', $user->getId()) - ->setPayload($response->output($clone, Response::MODEL_USER)); - } + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($clone); + + $queueForEvents + ->setParam('userId', $user->getId()) + ->setPayload($response->output($clone, Response::MODEL_USER)); $response->noContent(); }); diff --git a/docker-compose.yml b/docker-compose.yml index 2106ff8593..de71e3937a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 32f636a0f6..859fa6f97e 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Exception; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Document; use Utopia\Platform\Action; @@ -15,9 +16,7 @@ class Usage extends Action private int $lastTriggeredTime = 0; private int $keys = 0; private const INFINITY_PERIOD = '_inf_'; - private const KEYS_THRESHOLD = 5; - private const KEYS_SENT_THRESHOLD = 60; - + private const KEYS_THRESHOLD = 10000; public static function getName(): string @@ -57,7 +56,9 @@ class Usage extends Action if (empty($payload)) { throw new Exception('Missing payload'); } + //Todo Figure out way to preserve keys when the container is being recreated @shimonewman + $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -87,7 +88,7 @@ class Usage extends Action // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || - (time() - $this->lastTriggeredTime > self::KEYS_SENT_THRESHOLD && $this->keys > 0) + (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { $queueForUsageDump ->setStats($this->stats) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 24fe4ccccd..948b3fa681 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -55,6 +55,7 @@ class UsageDump extends Action throw new Exception('Missing payload'); } + //Todo rename both usage workers @shimonewman foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; From 72ee7d7f343485cdb27cb7229e2353a9dc17de9d Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 20:51:54 +0200 Subject: [PATCH 09/13] interval update --- .env | 2 +- src/Appwrite/Platform/Workers/Usage.php | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 3efd72e2a2..b915f91516 100644 --- a/.env +++ b/.env @@ -78,7 +78,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=60 +_APP_USAGE_AGGREGATION_INTERVAL=20 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 859fa6f97e..78efca3585 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers; use Exception; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; use Appwrite\Event\UsageDump; @@ -58,7 +59,7 @@ class Usage extends Action } //Todo Figure out way to preserve keys when the container is being recreated @shimonewman - $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60'); + $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -84,12 +85,12 @@ class Usage extends Action $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; } - // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { + var_dump(DateTime::now()); $queueForUsageDump ->setStats($this->stats) ->trigger(); From a1539395c636ab99df81ec71712e86e4b93b9a5d Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 21:08:29 +0200 Subject: [PATCH 10/13] interval update --- src/Appwrite/Platform/Workers/Usage.php | 3 ++- src/Appwrite/Platform/Workers/UsageDump.php | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 78efca3585..589e775e97 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -90,7 +90,8 @@ class Usage extends Action $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { - var_dump(DateTime::now()); + console::warning('[' . DateTime::now() . '] stats aggregation sent to worker with ' . $this->keys . ' keys'); + $queueForUsageDump ->setStats($this->stats) ->trigger(); diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 948b3fa681..ce221a2f62 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -59,12 +59,13 @@ class UsageDump extends Action foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $projectInternalId = $project->getInternalId(); if ($numberOfKeys === 0) { continue; } + console::warning('[' . DateTime::now() . '] aggregation received project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); + try { $dbForProject = $getProjectDB($project); foreach ($stats['keys'] ?? [] as $key => $value) { @@ -105,7 +106,7 @@ class UsageDump extends Action } } } catch (\Exception $e) { - console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); } } } From c1fb4ecf39d01dfd30f5376f141fc1c0648e62c6 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 21:41:23 +0200 Subject: [PATCH 11/13] interval update --- src/Appwrite/Platform/Workers/Usage.php | 2 +- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 589e775e97..8a9c322c80 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -90,7 +90,7 @@ class Usage extends Action $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { - console::warning('[' . DateTime::now() . '] stats aggregation sent to worker with ' . $this->keys . ' keys'); + console::log('[' . DateTime::now() . '] stats aggregation sent to worker with ' . $this->keys . ' keys'); $queueForUsageDump ->setStats($this->stats) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index ce221a2f62..a45cd40995 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -64,7 +64,7 @@ class UsageDump extends Action continue; } - console::warning('[' . DateTime::now() . '] aggregation received project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); + console::log('[' . DateTime::now() . '] aggregation received project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); try { $dbForProject = $getProjectDB($project); From 3788d5bcebc3eb7709d9bb4f3de6d6f500e5bd70 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 9 Feb 2024 12:56:37 +0200 Subject: [PATCH 12/13] interval update --- src/Appwrite/Platform/Workers/Usage.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 8a9c322c80..8490c31c0a 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -85,6 +85,10 @@ class Usage extends Action $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; } + var_dump(time() - $this->lastTriggeredTime > $aggregationInterval); + var_dump(time() - $this->lastTriggeredTime); + var_dump($aggregationInterval); + var_dump($this->keys); // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || From d0985db27dc00de6b953ac96e8d6c6a0e24abc30 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 9 Feb 2024 13:22:48 +0200 Subject: [PATCH 13/13] interval update --- src/Appwrite/Platform/Workers/Usage.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 8490c31c0a..d65ab0968d 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -85,10 +85,7 @@ class Usage extends Action $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; } - var_dump(time() - $this->lastTriggeredTime > $aggregationInterval); - var_dump(time() - $this->lastTriggeredTime); - var_dump($aggregationInterval); - var_dump($this->keys); + // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD ||