From 187ea93f814244707916025fdbd3c60cf19d0841 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 02:26:06 +0000 Subject: [PATCH 01/57] Get logs db resource --- app/cli.php | 33 +++++++++++++++++++++++++++++++++ app/init.php | 33 +++++++++++++++++++++++++++++++++ app/worker.php | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/app/cli.php b/app/cli.php index 47f4525f0b..dbd7c59784 100644 --- a/app/cli.php +++ b/app/cli.php @@ -160,6 +160,39 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform }; }, ['pools', 'dbForPlatform', 'cache']); +CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) { + $database = null; + return function (?Document $project = null) use ($pools, $cache, $database) { + if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + return $database; + } + + $dbAdapter = $pools + ->get('logs') + ->pop() + ->getResource(); + + $database = new Database( + $dbAdapter, + $cache + ); + + $database + ->setSharedTables(true) + ->setNamespace('logsV1') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + + // set tenant + if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + } + + return $database; + }; +}, ['pools', 'cache']); + CLI::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); diff --git a/app/init.php b/app/init.php index f812ef094c..651d62979a 100644 --- a/app/init.php +++ b/app/init.php @@ -1542,6 +1542,39 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform }; }, ['pools', 'dbForPlatform', 'cache']); +App::setResource('getLogsDB', function (Group $pools, Cache $cache) { + $database = null; + return function (?Document $project = null) use ($pools, $cache, $database) { + if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + return $database; + } + + $dbAdapter = $pools + ->get('logs') + ->pop() + ->getResource(); + + $database = new Database( + $dbAdapter, + $cache + ); + + $database + ->setSharedTables(true) + ->setNamespace('logsV1') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + + // set tenant + if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + } + + return $database; + }; +}, ['pools', 'cache']); + App::setResource('cache', function (Group $pools) { $list = Config::getParam('pools-cache', []); $adapters = []; diff --git a/app/worker.php b/app/worker.php index 6eb1363e9b..81b1367873 100644 --- a/app/worker.php +++ b/app/worker.php @@ -173,6 +173,39 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf }; }, ['pools', 'dbForPlatform', 'cache']); +Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { + $database = null; + return function (?Document $project = null) use ($pools, $cache, $database) { + if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + return $database; + } + + $dbAdapter = $pools + ->get('logs') + ->pop() + ->getResource(); + + $database = new Database( + $dbAdapter, + $cache + ); + + $database + ->setSharedTables(true) + ->setNamespace('logsV1') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + + // set tenant + if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + } + + return $database; + }; +}, ['pools', 'cache']); + Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); }); From c6360cce065f84abea056417683d3faf3ce9069f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 02:50:50 +0000 Subject: [PATCH 02/57] logs collection config --- app/config/collections.php | 2 + app/config/collections/logs.php | 94 +++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 app/config/collections/logs.php diff --git a/app/config/collections.php b/app/config/collections.php index 8c8356aafd..533dee57a8 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -5,6 +5,7 @@ $common = include __DIR__ . '/collections/common.php'; $projects = include __DIR__ . '/collections/projects.php'; $databases = include __DIR__ . '/collections/databases.php'; $platform = include __DIR__ . '/collections/platform.php'; +$logs = include __DIR__ . '/collections/logs.php'; // see - http.php#245 // $collections['buckets']['files']; @@ -27,6 +28,7 @@ $collections = [ 'databases' => $databases, 'projects' => array_merge($projects, $common), 'console' => array_merge($platform, $common), + 'logs' => $logs, ]; return $collections; diff --git a/app/config/collections/logs.php b/app/config/collections/logs.php new file mode 100644 index 0000000000..d6fba35722 --- /dev/null +++ b/app/config/collections/logs.php @@ -0,0 +1,94 @@ + ID::custom(Database::METADATA), + '$id' => ID::custom('stats'), + 'name' => 'Stats', + 'attributes' => [ + [ + '$id' => ID::custom('metric'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('region'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('value'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 8, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('time'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('period'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 4, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => ID::custom('_key_time'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => [Database::ORDER_DESC], + ], + [ + '$id' => ID::custom('_key_period_time'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['period', 'time'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_metric_period_time'), + 'type' => Database::INDEX_UNIQUE, + 'attributes' => ['metric', 'period', 'time'], + 'lengths' => [], + 'orders' => [Database::ORDER_DESC], + ], + ], +]; + +return $logsCollection; From 5685c8ac25917ff2a2200cb481f0ad4386d5951e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 02:51:01 +0000 Subject: [PATCH 03/57] refactor create collection and create logs db --- app/http.php | 251 ++++++++++++++++++++++++++------------------------- 1 file changed, 129 insertions(+), 122 deletions(-) diff --git a/app/http.php b/app/http.php index 608ac2ec12..b9aa69a7cc 100644 --- a/app/http.php +++ b/app/http.php @@ -17,7 +17,6 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; @@ -156,6 +155,79 @@ function dispatch(Server $server, int $fd, int $type, $data = null): int include __DIR__ . '/controllers/general.php'; +function createDatabase(App $app, string $resourceKey, string $dbName, array $collections, mixed $pools, callable $extraSetup = null): void +{ + $max = 10; + $sleep = 1; + $attempts = 0; + + do { + try { + $attempts++; + $resource = $app->getResource($resourceKey); + /* @var $database Database */ + $database = is_callable($resource) ? $resource() : $resource; + break; // exit loop on success + } catch (\Exception $e) { + Console::warning(" └── Database not ready. Retrying connection ({$attempts})..."); + $pools->reclaim(); + if ($attempts >= $max) { + throw new \Exception(' └── Failed to connect to database: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + Console::success("[Setup] - $dbName database init started..."); + + // Attempt to create the database + try { + Console::info(" └── Creating database: $dbName..."); + $database->create(); + } catch (\Exception $e) { + Console::info(" └── Skip: metadata table already exists"); + } + + // Process collections + foreach ($collections as $key => $collection) { + if (($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } + + if (!$database->getCollection($key)->isEmpty()) { + continue; + } + + Console::info(" └── Creating collection: {$collection['$id']}..."); + + $attributes = array_map(fn ($attr) => new Document([ + '$id' => ID::custom($attr['$id']), + 'type' => $attr['type'], + 'size' => $attr['size'], + 'required' => $attr['required'], + 'signed' => $attr['signed'], + 'array' => $attr['array'], + 'filters' => $attr['filters'], + 'default' => $attr['default'] ?? null, + 'format' => $attr['format'] ?? '' + ]), $collection['attributes']); + + $indexes = array_map(fn ($index) => new Document([ + '$id' => ID::custom($index['$id']), + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]), $collection['indexes']); + + $database->createCollection($key, $attributes, $indexes); + } + + if ($extraSetup) { + $extraSetup($database); + } +} + $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $register) { $app = new App('UTC'); @@ -164,140 +236,75 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg /** @var Group $pools */ App::setResource('pools', fn () => $pools); - // wait for database to be ready - $attempts = 0; - $max = 10; - $sleep = 1; - - do { - try { - $attempts++; - $dbForPlatform = $app->getResource('dbForPlatform'); - /** @var Utopia\Database\Database $dbForPlatform */ - break; // leave the do-while if successful - } catch (\Throwable $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < $max); - - Console::success('[Setup] - Server database init started...'); - - try { - Console::success('[Setup] - Creating console database...'); - $dbForPlatform->create(); - } catch (Duplicate) { - Console::success('[Setup] - Skip: metadata table already exists'); - } - - if ($dbForPlatform->getCollection(Audit::COLLECTION)->isEmpty()) { - $audit = new Audit($dbForPlatform); - $audit->setup(); - } - /** @var array $collections */ $collections = Config::getParam('collections', []); - $consoleCollections = $collections['console']; - foreach ($consoleCollections as $key => $collection) { - if (($collection['$collection'] ?? '') !== Database::METADATA) { - continue; - } - if (!$dbForPlatform->getCollection($key)->isEmpty()) { - continue; + + // create logs database first, `getLogsDB` is a callable. + createDatabase($app, 'getLogsDB', 'logs', $collections['logs'], $pools); + + // create appwrite database, `dbForPlatform` is a direct access call. + createDatabase($app, 'dbForPlatform', 'appwrite', $collections['console'], $pools, function (Database $dbForPlatform) use ($collections) { + if ($dbForPlatform->getCollection(Audit::COLLECTION)->isEmpty()) { + $audit = new Audit($dbForPlatform); + $audit->setup(); } - Console::success('[Setup] - Creating console collection: ' . $collection['$id'] . '...'); + if ($dbForPlatform->getDocument('buckets', 'default')->isEmpty() && + !$dbForPlatform->exists($dbForPlatform->getDatabase(), 'bucket_1')) { + Console::info(" └── Creating default bucket..."); + $dbForPlatform->createDocument('buckets', new Document([ + '$id' => ID::custom('default'), + '$collection' => ID::custom('buckets'), + 'name' => 'Default', + 'maximumFileSize' => (int) System::getEnv('_APP_STORAGE_LIMIT', 0), + 'allowedFileExtensions' => [], + 'enabled' => true, + 'compression' => 'gzip', + 'encryption' => true, + 'antivirus' => true, + 'fileSecurity' => true, + '$permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'search' => 'buckets Default', + ])); - $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); - $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); + $bucket = $dbForPlatform->getDocument('buckets', 'default'); - $dbForPlatform->createCollection($key, $attributes, $indexes); - } - - if ($dbForPlatform->getDocument('buckets', 'default')->isEmpty() && !$dbForPlatform->exists($dbForPlatform->getDatabase(), 'bucket_1')) { - Console::success('[Setup] - Creating default bucket...'); - $dbForPlatform->createDocument('buckets', new Document([ - '$id' => ID::custom('default'), - '$collection' => ID::custom('buckets'), - 'name' => 'Default', - 'maximumFileSize' => (int) System::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB - 'allowedFileExtensions' => [], - 'enabled' => true, - 'compression' => 'gzip', - 'encryption' => true, - 'antivirus' => true, - 'fileSecurity' => true, - '$permissions' => [ - Permission::create(Role::any()), - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'search' => 'buckets Default', - ])); - - $bucket = $dbForPlatform->getDocument('buckets', 'default'); - - Console::success('[Setup] - Creating files collection for default bucket...'); - $files = $collections['buckets']['files'] ?? []; - if (empty($files)) { - throw new Exception('Files collection is not configured.'); - } - - $attributes = \array_map(fn ($attribute) => new Document($attribute), $files['attributes']); - $indexes = \array_map(fn (array $index) => new Document($index), $files['indexes']); - - $dbForPlatform->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); - } - - $projectCollections = $collections['projects']; - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); - $sharedTablesV2 = \array_diff($sharedTables, $sharedTablesV1); - - $cache = $app->getResource('cache'); - - foreach ($sharedTablesV2 as $hostname) { - $adapter = $pools - ->get($hostname) - ->pop() - ->getResource(); - - $dbForProject = (new Database($adapter, $cache)) - ->setDatabase('appwrite') - ->setSharedTables(true) - ->setTenant(null) - ->setNamespace(System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '')); - - try { - Console::success('[Setup] - Creating project database: ' . $hostname . '...'); - $dbForProject->create(); - } catch (Duplicate) { - Console::success('[Setup] - Skip: metadata table already exists'); - } - - foreach ($projectCollections as $key => $collection) { - if (($collection['$collection'] ?? '') !== Database::METADATA) { - continue; - } - if (!$dbForProject->getCollection($key)->isEmpty()) { - continue; + Console::info(" └── Creating files collection for default bucket..."); + $files = $collections['buckets']['files'] ?? []; + if (empty($files)) { + throw new Exception('Files collection is not configured.'); } - $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); - $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); + $attributes = array_map(fn ($attr) => new Document([ + '$id' => ID::custom($attr['$id']), + 'type' => $attr['type'], + 'size' => $attr['size'], + 'required' => $attr['required'], + 'signed' => $attr['signed'], + 'array' => $attr['array'], + 'filters' => $attr['filters'], + 'default' => $attr['default'] ?? null, + 'format' => $attr['format'] ?? '' + ]), $files['attributes']); - Console::success('[Setup] - Creating project collection: ' . $collection['$id'] . '...'); + $indexes = array_map(fn ($index) => new Document([ + '$id' => ID::custom($index['$id']), + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]), $files['indexes']); - $dbForProject->createCollection($key, $attributes, $indexes); + $dbForPlatform->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } - } + }); $pools->reclaim(); - Console::success('[Setup] - Server database init completed...'); }); From 9955b6b61c0e7fc5fdfab48f7c74e24a246f2635 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 04:33:33 +0000 Subject: [PATCH 04/57] usage count container and events --- .env | 1 + Dockerfile | 3 +- app/cli.php | 4 + bin/usage-count | 3 + docker-compose.yml | 32 +++++++ src/Appwrite/Event/Usage.php | 12 +++ src/Appwrite/Platform/Action.php | 90 ++++++++++++++++++ src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/UsageCount.php | 101 +++++++++++++++++++++ 9 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 bin/usage-count create mode 100644 src/Appwrite/Platform/Action.php create mode 100644 src/Appwrite/Platform/Tasks/UsageCount.php diff --git a/.env b/.env index 8f7d7996e7..58cd9c13fb 100644 --- a/.env +++ b/.env @@ -86,6 +86,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_USAGE_AGGREGATION_INTERVAL=30 +_APP_USAGE_COUNT_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/Dockerfile b/Dockerfile index 41810f5dc4..30554289ef 100755 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-usage && \ - chmod +x /usr/local/bin/worker-usage-dump + chmod +x /usr/local/bin/worker-usage-dump && \ + chmod +x /usr/local/bin/usage-count # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/cli.php b/app/cli.php index dbd7c59784..91fef8dc55 100644 --- a/app/cli.php +++ b/app/cli.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/init.php'; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; +use Appwrite\Event\Usage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; use Utopia\Cache\Adapter\Sharding; @@ -205,6 +206,9 @@ CLI::setResource('queueForDeletes', function (Connection $queue) { CLI::setResource('queueForCertificates', function (Connection $queue) { return new Certificate($queue); }, ['queue']); +CLI::setResource('queueForUsage', function (Connection $queue) { + return new Usage($queue); +}, ['queue']); CLI::setResource('logError', function (Registry $register) { return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); diff --git a/bin/usage-count b/bin/usage-count new file mode 100644 index 0000000000..c5d696cbd4 --- /dev/null +++ b/bin/usage-count @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage-count $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2fb19f7126..44d5fb6d01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -722,6 +722,38 @@ services: - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES + appwrite-task-usage-count: + container_name: appwrite-task-usage-count + entrypoint: usage-count + <<: *x-logging + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_DATABASE_SHARED_TABLES + - _APP_USAGE_COUNT_INTERVAL + appwrite-worker-usage: entrypoint: worker-usage <<: *x-logging diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 5609859f37..e7423362cd 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -7,9 +7,14 @@ use Utopia\Queue\Connection; class Usage extends Event { + public const TYPE_USAGE_DUMP = 'usage-dump'; + public const TYPE_USAGE_COUNT = 'usage-count'; + protected array $metrics = []; protected array $reduce = []; + protected string $type = self::TYPE_USAGE_DUMP; + public function __construct(protected Connection $connection) { parent::__construct($connection); @@ -19,6 +24,12 @@ class Usage extends Event ->setClass(Event::USAGE_CLASS_NAME); } + public function setType(string $type): self + { + $this->type = $type; + return $this; + } + /** * Add reduce. * @@ -61,6 +72,7 @@ class Usage extends Event 'project' => $this->project, 'reduce' => $this->reduce, 'metrics' => $this->metrics, + 'type' => $this->type, ]; } diff --git a/src/Appwrite/Platform/Action.php b/src/Appwrite/Platform/Action.php new file mode 100644 index 0000000000..72c41582ea --- /dev/null +++ b/src/Appwrite/Platform/Action.php @@ -0,0 +1,90 @@ +disableValidation(); + $results = $database->find($collection, $newQueries); + $database->enableValidation(); + } catch (\Exception $e) { + if (!empty($this->logError)) { + call_user_func_array($this->logError, [$e, "CLI", "fetch_documents_namespace_{$database->getNamespace()}_collection{$collection}"]); + } + } + + if (empty($results)) { + return; + } + + $sum = count($results); + + if ($concurrent) { + $callables = []; + $errors = []; + + foreach ($results as $document) { + if (is_callable($callback)) { + $callables[] = Co\go(function () use ($document, $callback, &$errors) { + try { + $callback($document); + } catch (\Throwable $error) { + $errors[] = $error; + } + }); + } + } + + Co::join($callables); + + if (!empty($errors)) { + throw new \Error("Errors found in concurrent foreachDocument: " . \json_encode($errors)); + } + } else { + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + } + } + + $latestDocument = $results[array_key_last($results)]; + } + } +} diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 7a0d5b60ac..b2f1000f3e 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -15,6 +15,7 @@ use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Upgrade; +use Appwrite\Platform\Tasks\UsageCount; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Utopia\Platform\Service; @@ -40,6 +41,7 @@ class Tasks extends Service ->addAction(Upgrade::getName(), new Upgrade()) ->addAction(Vars::getName(), new Vars()) ->addAction(Version::getName(), new Version()) + ->addAction(UsageCount::getName(), new UsageCount()) ; } } diff --git a/src/Appwrite/Platform/Tasks/UsageCount.php b/src/Appwrite/Platform/Tasks/UsageCount.php new file mode 100644 index 0000000000..ef25a4d3f5 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/UsageCount.php @@ -0,0 +1,101 @@ +desc('Schedules projects for usage count') + ->inject('dbForPlatform') + ->inject('logError') + ->inject('queueForUsage') + ->callback(fn ($dbForPlatform, $logError, $queueForUsage) => $this->action($dbForPlatform, $logError, $queueForUsage)); + } + + public function action(Database $dbForPlatform, callable $logError, Usage $queueForUsage): void + { + $this->logError = $logError; + $this->dbForPlatform = $dbForPlatform; + $this->queue = $queueForUsage; + + Console::title("Usage count V1"); + + Console::success('Usage count: Started'); + + $interval = (int) System::getEnv('_APP_USAGE_COUNT_INTERVAL', '3600'); + Console::loop(function () use ($queueForUsage) { + $this->enqueueProjects($queueForUsage); + }, $interval); + + Console::log("Usage count: Exited"); + } + + /** + * Enqueue projects for counting + * @param Database $dbForPlatform + * @param Usage $queue + * @return void + */ + protected function enqueueProjects(Usage $queue): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); + // Foreach Team + $this->foreachDocument($this->dbForPlatform, 'projects', [ + Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours)) + ], function ($project) use ($queue) { + $queue + ->setProject($project) + ->setType(Usage::TYPE_USAGE_COUNT) + ->trigger(); + Console::success('project: ' . $project->getId() . '(' . $project->getInternalId() . ')' . ' queued'); + }); + } +} From 066273d0db84bd0121762b2e3a4323a6ff4b787f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 04:33:41 +0000 Subject: [PATCH 05/57] update usage worker for usage count --- app/init.php | 10 ++ src/Appwrite/Platform/Workers/Usage.php | 230 +++++++++++++++++++++++- 2 files changed, 238 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 651d62979a..8a376823ce 100644 --- a/app/init.php +++ b/app/init.php @@ -301,6 +301,16 @@ const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.execution const METRIC_NETWORK_REQUESTS = 'network.requests'; const METRIC_NETWORK_INBOUND = 'network.inbound'; const METRIC_NETWORK_OUTBOUND = 'network.outbound'; +const METRIC_USERS_ACTIVE = 'users.mau'; +const METRIC_WEBHOOKS = 'webhooks'; +const METRIC_PLATFORMS = 'platforms'; +const METRIC_PROVIDERS = 'providers'; +const METRIC_TOPICS = 'topics'; +const METRIC_KEYS = 'keys'; +const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; +const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; +const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; +const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; // Resource types diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3f7428d0dd..24fcc1bb3b 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -2,17 +2,27 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Event\Usage as UsageEvent; use Appwrite\Event\UsageDump; +use Appwrite\Platform\Action; use Exception; +use Throwable; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Platform\Action; +use Utopia\Database\Query; use Utopia\Queue\Message; use Utopia\System\System; class Usage extends Action { + /** + * Log Error Callback + * + * @var callable + */ + protected mixed $logError; private array $stats = []; private int $lastTriggeredTime = 0; private int $aggregationInterval = 20; @@ -36,6 +46,9 @@ class Usage extends Action ->inject('project') ->inject('getProjectDB') ->inject('queueForUsageDump') + ->inject('getLogsDB') + ->inject('dbForPlatform') + ->inject('logError') ->callback([$this, 'action']); $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); @@ -51,8 +64,10 @@ class Usage extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump): void + public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump, callable $getLogsDB, Database $dbForPlatform, callable $logError): void { + $this->logError = $logError; + $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); @@ -63,6 +78,12 @@ class Usage extends Action return; } + $type = $payload['type'] ?? UsageEvent::TYPE_USAGE_DUMP; + if ($type === UsageEvent::TYPE_USAGE_COUNT) { + $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); + return; + } + $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { if (empty($document)) { @@ -285,4 +306,209 @@ class Usage extends Action console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } + + + protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void + { + Console::info('Begining count for: ' . $project->getId()); + + try { + /** @var \Utopia\Database\Database $dbForLogs */ + $dbForLogs = call_user_func($getLogsDB, $project); + /** @var \Utopia\Database\Database $dbForProject */ + $dbForProject = call_user_func($getProjectDB, $project); + + $region = $project->getAttribute('region'); + + $platforms = $dbForPlatform->count('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $webhooks = $dbForPlatform->count('webhooks', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $keys = $dbForPlatform->count('keys', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $databases = $dbForProject->count('databases'); + $buckets = $dbForProject->count('buckets'); + $users = $dbForProject->count('users'); + + $last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00'); + $usersActive = $dbForProject->count('users', [ + Query::greaterThanEqual('accessedAt', $last30Days) + ]); + $teams = $dbForProject->count('teams'); + $functions = $dbForProject->count('functions'); + $messages = $dbForProject->count('messages'); + $providers = $dbForProject->count('providers'); + $topics = $dbForProject->count('topics'); + + $metrics = [ + METRIC_DATABASES => $databases, + METRIC_BUCKETS => $buckets, + METRIC_USERS => $users, + METRIC_FUNCTIONS => $functions, + METRIC_TEAMS => $teams, + METRIC_MESSAGES => $messages, + METRIC_USERS_ACTIVE => $usersActive, + METRIC_WEBHOOKS => $webhooks, + METRIC_PLATFORMS => $platforms, + METRIC_PROVIDERS => $providers, + METRIC_TOPICS => $topics, + METRIC_KEYS => $keys, + ]; + + foreach ($metrics as $metric => $value) { + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $value); + } + + try { + $this->countForBuckets($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "usageCount", "count_for_buckets_{$project->getId()}"]); + } + + try { + $this->countForDatabase($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "usageCount", "count_for_database_{$project->getId()}"]); + } + + try { + $this->countForFunctions($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "usageCount", "count_for_functions_{$project->getId()}"]); + } + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "usageCount", "count_for_project_{$project->getId()}"]); + } + + Console::info('End of count for: ' . $project->getId()); + } + + protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region) + { + $totalFiles = 0; + $totalStorage = 0; + $this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) { + $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); + + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $files); + + $storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual'); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $storage); + + $totalStorage += $storage; + $totalFiles += $files; + }); + + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES, $totalFiles); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES_STORAGE, $totalStorage); + } + + protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region) + { + $totalCollections = 0; + $totalDocuments = 0; + + $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) { + $collections = $dbForProject->count('database_' . $database->getInternalId()); + + $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $collections); + + $documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region); + + $totalDocuments += $documents; + $totalCollections += $collections; + }); + + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_COLLECTIONS, $totalCollections); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DOCUMENTS, $totalDocuments); + } + protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int + { + $databaseDocuments = 0; + $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) { + $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + + $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $documents); + + $databaseDocuments += $documents; + }); + + $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $databaseDocuments); + + return $databaseDocuments; + } + + protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) + { + $deploymentsStorage = $dbForProject->sum('deployments', 'size'); + $buildsStorage = $dbForProject->sum('builds', 'size'); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS_STORAGE, $buildsStorage); + + $deployments = $dbForProject->count('deployments'); + $builds = $dbForProject->count('builds'); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS, $deployments); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS, $builds); + + + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { + $functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]); + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage); + + $functionDeployments = $dbForProject->count('deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]); + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments); + + /** + * As deployments and builds have 1-1 relationship, + * the count for one should match the other + */ + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments); + + $functionBuildsStorage = 0; + + $this->foreachDocument($dbForProject, 'deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void { + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $functionBuildsStorage += $build->getAttribute('size', 0); + }); + + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); + }); + } + + protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value) + { + $id = md5(self::INFINITY_PERIOD . $metric); + $current = $dbForLogs->getDocument('stats', $id); + + if ($current->isEmpty()) { + $dbForLogs->createDocument('stats', new Document([ + '$id' => $id, + 'metric' => $metric, + 'period' => 'inf', + 'region' => $region, + 'value' => $value, + ])); + Console::success('Usage logs created for metric: ' . $metric); + } else { + $dbForLogs->updateDocument('stats', $id, $current->setAttribute('value', $value)); + Console::success('Usage logs updated for metric: ' . $metric); + } + } } From 5d5547c6b9adff10654187a7dfd584505d3bcba1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 06:11:58 +0000 Subject: [PATCH 06/57] periodic metric --- src/Appwrite/Platform/Workers/Usage.php | 42 +++++++++++++++---------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 24fcc1bb3b..a88023e175 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -17,6 +17,14 @@ use Utopia\System\System; class Usage extends Action { + /** + * Date format for different periods + */ + protected array $periods = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; /** * Log Error Callback * @@ -493,22 +501,24 @@ class Usage extends Action } protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value) - { - $id = md5(self::INFINITY_PERIOD . $metric); - $current = $dbForLogs->getDocument('stats', $id); - - if ($current->isEmpty()) { - $dbForLogs->createDocument('stats', new Document([ - '$id' => $id, - 'metric' => $metric, - 'period' => 'inf', - 'region' => $region, - 'value' => $value, - ])); - Console::success('Usage logs created for metric: ' . $metric); - } else { - $dbForLogs->updateDocument('stats', $id, $current->setAttribute('value', $value)); - Console::success('Usage logs updated for metric: ' . $metric); + { + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : \date($format, \time()); + $id = \md5("{$time}_{$period}_{$metric}"); + $current = $dbForLogs->getDocument('stats', $id); + if ($current->isEmpty()) { + $dbForLogs->createDocument('stats', new Document([ + '$id' => $id, + 'metric' => $metric, + 'period' => $period, + 'region' => $region, + 'value' => $value, + ])); + Console::success('Usage logs created for metric: ' . $metric . ' period:'. $period); + } else { + $dbForLogs->updateDocument('stats', $id, $current->setAttribute('value', $value)); + Console::success('Usage logs updated for metric: ' . $metric . ' period:'. $period); + } } } } From 55cbab62703c488db742ad339511aee06f48b20b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 06:15:59 +0000 Subject: [PATCH 07/57] fix format --- src/Appwrite/Platform/Workers/Usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index a88023e175..f6219e5ff8 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -501,7 +501,7 @@ class Usage extends Action } protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value) - { + { foreach ($this->periods as $period => $format) { $time = 'inf' === $period ? null : \date($format, \time()); $id = \md5("{$time}_{$period}_{$metric}"); From f593498c07100c9e68cada826ac2bf142ca11912 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 07:15:06 +0000 Subject: [PATCH 08/57] dual writing --- src/Appwrite/Platform/Workers/UsageDump.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index bb1d605442..4feaf650f8 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -38,6 +38,7 @@ class UsageDump extends Action $this ->inject('message') ->inject('getProjectDB') + ->inject('getLogsDB') ->callback([$this, 'action']); } @@ -48,7 +49,7 @@ class UsageDump extends Action * @throws Exception * @throws \Utopia\Database\Exception */ - public function action(Message $message, callable $getProjectDB): void + public function action(Message $message, callable $getProjectDB, callable $getLogsDB): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -65,6 +66,7 @@ class UsageDump extends Action } $dbForProject = $getProjectDB($project); + $dbForLogs = $getLogsDB($dbForProject); $projectDocuments = []; $databaseCache = []; $collectionSizeCache = []; @@ -112,6 +114,11 @@ class UsageDump extends Action documents: $projectDocuments ); + $dbForLogs->createOrUpdateDocumentsWithIncrease( + collection: 'stats', + attribute: 'value', + documents: $projectDocuments + ); $end = \microtime(true); Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s'); } From 336b0a7c02ef67a4e392f71e3353d22ac4d92a97 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 07:22:50 +0000 Subject: [PATCH 09/57] rename collection --- app/config/collections/logs.php | 6 +++--- src/Appwrite/Platform/Workers/Usage.php | 6 +++--- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/config/collections/logs.php b/app/config/collections/logs.php index d6fba35722..6beb6a8b32 100644 --- a/app/config/collections/logs.php +++ b/app/config/collections/logs.php @@ -5,10 +5,10 @@ use Utopia\Database\Helpers\ID; $logsCollection = []; -$logsCollection['stats'] = [ +$logsCollection['usage'] = [ '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('stats'), - 'name' => 'Stats', + '$id' => ID::custom('usage'), + 'name' => 'usage', 'attributes' => [ [ '$id' => ID::custom('metric'), diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index f6219e5ff8..801d8117f7 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -505,9 +505,9 @@ class Usage extends Action foreach ($this->periods as $period => $format) { $time = 'inf' === $period ? null : \date($format, \time()); $id = \md5("{$time}_{$period}_{$metric}"); - $current = $dbForLogs->getDocument('stats', $id); + $current = $dbForLogs->getDocument('usage', $id); if ($current->isEmpty()) { - $dbForLogs->createDocument('stats', new Document([ + $dbForLogs->createDocument('usage', new Document([ '$id' => $id, 'metric' => $metric, 'period' => $period, @@ -516,7 +516,7 @@ class Usage extends Action ])); Console::success('Usage logs created for metric: ' . $metric . ' period:'. $period); } else { - $dbForLogs->updateDocument('stats', $id, $current->setAttribute('value', $value)); + $dbForLogs->updateDocument('usage', $id, $current->setAttribute('value', $value)); Console::success('Usage logs updated for metric: ' . $metric . ' period:'. $period); } } diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 4feaf650f8..833d213a48 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -115,7 +115,7 @@ class UsageDump extends Action ); $dbForLogs->createOrUpdateDocumentsWithIncrease( - collection: 'stats', + collection: 'usage', attribute: 'value', documents: $projectDocuments ); From 3124554403b503871e4f5904d4c4ff476c107787 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 07:39:33 +0000 Subject: [PATCH 10/57] remove db listeners as counters are now done separately by usage worker --- app/controllers/shared/api.php | 115 +-------------------------------- 1 file changed, 1 insertion(+), 114 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e2845521f8..a3a41e33ed 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -90,98 +90,6 @@ $eventDatabaseListener = function (Document $project, Document $document, Respon } }; -$usageDatabaseListener = function (string $event, Document $document, Usage $queueForUsage) { - $value = 1; - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $value = -1; - } - - switch (true) { - case $document->getCollection() === 'teams': - $queueForUsage - ->addMetric(METRIC_TEAMS, $value); // per project - break; - case $document->getCollection() === 'users': - $queueForUsage - ->addMetric(METRIC_USERS, $value); // per project - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case $document->getCollection() === 'sessions': // sessions - $queueForUsage - ->addMetric(METRIC_SESSIONS, $value); //per project - break; - case $document->getCollection() === 'databases': // databases - $queueForUsage - ->addMetric(METRIC_DATABASES, $value); // per project - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $queueForUsage - ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) - ; - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $collectionInternalId = $parts[3] ?? 0; - $queueForUsage - ->addMetric(METRIC_DOCUMENTS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection - break; - case $document->getCollection() === 'buckets': //buckets - $queueForUsage - ->addMetric(METRIC_BUCKETS, $value); // per project - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case str_starts_with($document->getCollection(), 'bucket_'): // files - $parts = explode('_', $document->getCollection()); - $bucketInternalId = $parts[1]; - $queueForUsage - ->addMetric(METRIC_FILES, $value) // per project - ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project - ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket - ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket - break; - case $document->getCollection() === 'functions': - $queueForUsage - ->addMetric(METRIC_FUNCTIONS, $value); // per project - - if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage - ->addReduce($document); - } - break; - case $document->getCollection() === 'deployments': - $queueForUsage - ->addMetric(METRIC_DEPLOYMENTS, $value) // per project - ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value); - break; - default: - break; - } -}; - App::init() ->groups(['api']) ->inject('utopia') @@ -436,11 +344,10 @@ App::init() ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') - ->inject('queueForUsage') ->inject('dbForProject') ->inject('timelimit') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Database $dbForProject, callable $timelimit, string $mode) { $route = $utopia->getRoute(); @@ -542,26 +449,6 @@ App::init() $queueForBuilds->setProject($project); $queueForMessaging->setProject($project); - // Clone the queues, to prevent events triggered by the database listener - // from overwriting the events that are supposed to be triggered in the shutdown hook. - $queueForEventsClone = new Event($queue); - $queueForFunctions = new Func($queue); - $queueForWebhooks = new Webhook($queue); - $queueForRealtime = new Realtime(); - - $dbForProject - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) - ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( - $project, - $document, - $response, - $queueForEventsClone->from($queueForEvents), - $queueForFunctions->from($queueForEvents), - $queueForWebhooks->from($queueForEvents), - $queueForRealtime->from($queueForEvents) - )); - $useCache = $route->getLabel('cache', false); if ($useCache) { $key = md5($request->getURI() . '*' . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); From 05f99dcdf007165e50ec03184f65a5a155965927 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 07:46:54 +0000 Subject: [PATCH 11/57] fix wrong arg --- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 833d213a48..0c83a47bfc 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -66,7 +66,7 @@ class UsageDump extends Action } $dbForProject = $getProjectDB($project); - $dbForLogs = $getLogsDB($dbForProject); + $dbForLogs = $getLogsDB($project); $projectDocuments = []; $databaseCache = []; $collectionSizeCache = []; From 7b588219f914424ff05b9e7b430a64c1f38b08c0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 27 Jan 2025 08:19:03 +0000 Subject: [PATCH 12/57] fix dual writing internal ID conflict --- src/Appwrite/Platform/Workers/UsageDump.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 0c83a47bfc..6e7c59d200 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -108,6 +108,17 @@ class UsageDump extends Action } } + /** + * Create clone to dual write + * This is required as first request to db + * modifies the document's details like internal ID + * which will conflict in new DB + */ + $clonedProjectDoucments = []; + foreach ($projectDocuments as $document) { + $clonedProjectDoucments[] = new Document($document->getArrayCopy()); + } + $dbForProject->createOrUpdateDocumentsWithIncrease( collection: 'stats', attribute: 'value', @@ -117,7 +128,7 @@ class UsageDump extends Action $dbForLogs->createOrUpdateDocumentsWithIncrease( collection: 'usage', attribute: 'value', - documents: $projectDocuments + documents: $clonedProjectDoucments ); $end = \microtime(true); Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s'); From d84f6e4ee0fc942119c819ce69ae295f077237ce Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 28 Jan 2025 06:40:20 +0000 Subject: [PATCH 13/57] reset database listeners --- app/controllers/shared/api.php | 115 ++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a3a41e33ed..e2845521f8 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -90,6 +90,98 @@ $eventDatabaseListener = function (Document $project, Document $document, Respon } }; +$usageDatabaseListener = function (string $event, Document $document, Usage $queueForUsage) { + $value = 1; + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $value = -1; + } + + switch (true) { + case $document->getCollection() === 'teams': + $queueForUsage + ->addMetric(METRIC_TEAMS, $value); // per project + break; + case $document->getCollection() === 'users': + $queueForUsage + ->addMetric(METRIC_USERS, $value); // per project + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case $document->getCollection() === 'sessions': // sessions + $queueForUsage + ->addMetric(METRIC_SESSIONS, $value); //per project + break; + case $document->getCollection() === 'databases': // databases + $queueForUsage + ->addMetric(METRIC_DATABASES, $value); // per project + + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $queueForUsage + ->addMetric(METRIC_COLLECTIONS, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) + ; + + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $collectionInternalId = $parts[3] ?? 0; + $queueForUsage + ->addMetric(METRIC_DOCUMENTS, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection + break; + case $document->getCollection() === 'buckets': //buckets + $queueForUsage + ->addMetric(METRIC_BUCKETS, $value); // per project + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case str_starts_with($document->getCollection(), 'bucket_'): // files + $parts = explode('_', $document->getCollection()); + $bucketInternalId = $parts[1]; + $queueForUsage + ->addMetric(METRIC_FILES, $value) // per project + ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project + ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket + ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket + break; + case $document->getCollection() === 'functions': + $queueForUsage + ->addMetric(METRIC_FUNCTIONS, $value); // per project + + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case $document->getCollection() === 'deployments': + $queueForUsage + ->addMetric(METRIC_DEPLOYMENTS, $value) // per project + ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value) // per function + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value); + break; + default: + break; + } +}; + App::init() ->groups(['api']) ->inject('utopia') @@ -344,10 +436,11 @@ App::init() ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') + ->inject('queueForUsage') ->inject('dbForProject') ->inject('timelimit') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Database $dbForProject, callable $timelimit, string $mode) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -449,6 +542,26 @@ App::init() $queueForBuilds->setProject($project); $queueForMessaging->setProject($project); + // Clone the queues, to prevent events triggered by the database listener + // from overwriting the events that are supposed to be triggered in the shutdown hook. + $queueForEventsClone = new Event($queue); + $queueForFunctions = new Func($queue); + $queueForWebhooks = new Webhook($queue); + $queueForRealtime = new Realtime(); + + $dbForProject + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) + ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( + $project, + $document, + $response, + $queueForEventsClone->from($queueForEvents), + $queueForFunctions->from($queueForEvents), + $queueForWebhooks->from($queueForEvents), + $queueForRealtime->from($queueForEvents) + )); + $useCache = $route->getLabel('cache', false); if ($useCache) { $key = md5($request->getURI() . '*' . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); From fb5aeb14764e82df41fe2306516cf089195d64de Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 28 Jan 2025 07:19:27 +0000 Subject: [PATCH 14/57] skip some metrics override --- src/Appwrite/Platform/Workers/UsageDump.php | 53 +++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 6e7c59d200..0e7103b8c2 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -19,6 +19,51 @@ const METRIC_PROJECT_LEVEL_STORAGE = 2; class UsageDump extends Action { protected array $stats = []; + + /** + * Metrics to skip writing to logsDB + * As these metrics are calculated separately + * by logs DB + * @var array + */ + protected array $skipBaseMetrics = [ + METRIC_DATABASES => true, + METRIC_BUCKETS => true, + METRIC_USERS => true, + METRIC_FUNCTIONS => true, + METRIC_TEAMS => true, + METRIC_MESSAGES => true, + METRIC_USERS_ACTIVE => true, + METRIC_WEBHOOKS => true, + METRIC_PLATFORMS => true, + METRIC_PROVIDERS => true, + METRIC_TOPICS => true, + METRIC_KEYS => true, + METRIC_FILES => true, + METRIC_FILES_STORAGE => true, + METRIC_DEPLOYMENTS_STORAGE => true, + METRIC_BUILDS_STORAGE => true, + METRIC_DEPLOYMENTS => true, + METRIC_BUILDS => true, + METRIC_COLLECTIONS => true, + METRIC_DOCUMENTS => true, + ]; + + /** + * Skip metrics associated with parent IDs + * these need to be checked individually with `str_ends_with` + */ + protected array $skipParentIdMetrics = [ + '.files', + '.files.storage', + '.collections', + '.documents', + '.deployments', + '.deployments.storage', + '.builds', + '.builds.storage', + ]; + protected array $periods = [ '1h' => 'Y-m-d H:00', '1d' => 'Y-m-d 00:00', @@ -116,6 +161,14 @@ class UsageDump extends Action */ $clonedProjectDoucments = []; foreach ($projectDocuments as $document) { + if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) { + continue; + } + foreach ($this->skipParentIdMetrics as $skipMetric) { + if (str_ends_with($document->getAttribute('metric'), $skipMetric)) { + continue; + } + } $clonedProjectDoucments[] = new Document($document->getArrayCopy()); } From 21097041f0513c3332e66a110b78a55dc28f337d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 03:50:27 +0000 Subject: [PATCH 15/57] refactor: rename usage count to stats resources --- Dockerfile | 2 +- bin/stats-resources | 3 +++ bin/usage-count | 3 --- docker-compose.yml | 6 +++--- src/Appwrite/Event/Usage.php | 2 +- src/Appwrite/Platform/Services/Tasks.php | 4 ++-- .../Platform/Tasks/{UsageCount.php => StatsResources.php} | 4 ++-- src/Appwrite/Platform/Workers/Usage.php | 8 ++++---- 8 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 bin/stats-resources delete mode 100644 bin/usage-count rename src/Appwrite/Platform/Tasks/{UsageCount.php => StatsResources.php} (97%) diff --git a/Dockerfile b/Dockerfile index 30554289ef..c38e0493fe 100755 --- a/Dockerfile +++ b/Dockerfile @@ -87,7 +87,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-usage-dump && \ - chmod +x /usr/local/bin/usage-count + chmod +x /usr/local/bin/stats-resources # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/stats-resources b/bin/stats-resources new file mode 100644 index 0000000000..3104bab896 --- /dev/null +++ b/bin/stats-resources @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php stats-resources $@ \ No newline at end of file diff --git a/bin/usage-count b/bin/usage-count deleted file mode 100644 index c5d696cbd4..0000000000 --- a/bin/usage-count +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php usage-count $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 44d5fb6d01..d294896adc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -722,9 +722,9 @@ services: - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES - appwrite-task-usage-count: - container_name: appwrite-task-usage-count - entrypoint: usage-count + appwrite-task-stats-resources: + container_name: appwrite-task-stats-resources + entrypoint: stats-resources <<: *x-logging image: appwrite-dev networks: diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index e7423362cd..7d630bf122 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -8,7 +8,7 @@ use Utopia\Queue\Connection; class Usage extends Event { public const TYPE_USAGE_DUMP = 'usage-dump'; - public const TYPE_USAGE_COUNT = 'usage-count'; + public const TYPE_USAGE_COUNT = 'stats-resources'; protected array $metrics = []; protected array $reduce = []; diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index b2f1000f3e..2622bd18a2 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -15,7 +15,7 @@ use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Upgrade; -use Appwrite\Platform\Tasks\UsageCount; +use Appwrite\Platform\Tasks\StatsResources; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Utopia\Platform\Service; @@ -41,7 +41,7 @@ class Tasks extends Service ->addAction(Upgrade::getName(), new Upgrade()) ->addAction(Vars::getName(), new Vars()) ->addAction(Version::getName(), new Version()) - ->addAction(UsageCount::getName(), new UsageCount()) + ->addAction(StatsResources::getName(), new StatsResources()) ; } } diff --git a/src/Appwrite/Platform/Tasks/UsageCount.php b/src/Appwrite/Platform/Tasks/StatsResources.php similarity index 97% rename from src/Appwrite/Platform/Tasks/UsageCount.php rename to src/Appwrite/Platform/Tasks/StatsResources.php index ef25a4d3f5..5f077bbe9f 100644 --- a/src/Appwrite/Platform/Tasks/UsageCount.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -19,7 +19,7 @@ const INFINITY_PERIOD = '_inf_'; * Runs every hour, schedules project * for aggregating resource count */ -class UsageCount extends Action +class StatsResources extends Action { /** * Log Error Callback @@ -44,7 +44,7 @@ class UsageCount extends Action public static function getName() { - return 'usage-count'; + return 'stats-resources'; } public function __construct() diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 801d8117f7..039dad939b 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -373,22 +373,22 @@ class Usage extends Action try { $this->countForBuckets($dbForProject, $dbForLogs, $region); } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "usageCount", "count_for_buckets_{$project->getId()}"]); + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); } try { $this->countForDatabase($dbForProject, $dbForLogs, $region); } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "usageCount", "count_for_database_{$project->getId()}"]); + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]); } try { $this->countForFunctions($dbForProject, $dbForLogs, $region); } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "usageCount", "count_for_functions_{$project->getId()}"]); + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]); } } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "usageCount", "count_for_project_{$project->getId()}"]); + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); } Console::info('End of count for: ' . $project->getId()); From 31ddb6e48f1d383bcc995a5db0ab117aa892ac48 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 03:51:17 +0000 Subject: [PATCH 16/57] rename collection --- app/config/collections/logs.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/config/collections/logs.php b/app/config/collections/logs.php index 6beb6a8b32..069dcf5a4b 100644 --- a/app/config/collections/logs.php +++ b/app/config/collections/logs.php @@ -5,10 +5,10 @@ use Utopia\Database\Helpers\ID; $logsCollection = []; -$logsCollection['usage'] = [ +$logsCollection['stats'] = [ '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('usage'), - 'name' => 'usage', + '$id' => ID::custom('stats'), + 'name' => 'stats', 'attributes' => [ [ '$id' => ID::custom('metric'), From 22ab23d009cfb0af9edc397baaf396734a0b37d2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:05:51 +0000 Subject: [PATCH 17/57] feat: Stats resources worker --- Dockerfile | 3 +- bin/worker-stats-resources | 3 + docker-compose.yml | 31 ++ src/Appwrite/Platform/Services/Tasks.php | 2 +- src/Appwrite/Platform/Services/Workers.php | 3 +- .../Platform/Workers/StatsResources.php | 278 ++++++++++++++++++ 6 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 bin/worker-stats-resources create mode 100644 src/Appwrite/Platform/Workers/StatsResources.php diff --git a/Dockerfile b/Dockerfile index c38e0493fe..1d50684535 100755 --- a/Dockerfile +++ b/Dockerfile @@ -87,7 +87,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-usage-dump && \ - chmod +x /usr/local/bin/stats-resources + chmod +x /usr/local/bin/stats-resources && \ + chmod +x /usr/local/bin/worker-stats-resources # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/worker-stats-resources b/bin/worker-stats-resources new file mode 100644 index 0000000000..9c5d2bebff --- /dev/null +++ b/bin/worker-stats-resources @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php stats-resources $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d294896adc..d6345abddb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -754,6 +754,37 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_USAGE_COUNT_INTERVAL + appwrite-worker-stats-resources: + entrypoint: worker-stats-resources + <<: *x-logging + container_name: appwrite-worker-stats-resources + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_DATABASE_SHARED_TABLES + appwrite-worker-usage: entrypoint: worker-usage <<: *x-logging diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2622bd18a2..eb6d775da9 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -14,8 +14,8 @@ use Appwrite\Platform\Tasks\ScheduleMessages; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; -use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\StatsResources; +use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Utopia\Platform\Service; diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 0e79f4257c..150c8cfd23 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -11,6 +11,7 @@ use Appwrite\Platform\Workers\Functions; use Appwrite\Platform\Workers\Mails; use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; +use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\Usage; use Appwrite\Platform\Workers\UsageDump; use Appwrite\Platform\Workers\Webhooks; @@ -34,7 +35,7 @@ class Workers extends Service ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) - + ->addAction(StatsResources::getName(), new StatsResources()) ; } } diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php new file mode 100644 index 0000000000..e4ba59c464 --- /dev/null +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -0,0 +1,278 @@ + 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'stats-resources'; + } + + /** + * @throws Exception + */ + public function __construct() + { + $this + ->desc('Stats resources worker') + ->inject('message') + ->inject('project') + ->inject('getProjectDB') + ->inject('getLogsDB') + ->inject('dbForPlatform') + ->inject('logError') + ->callback([$this, 'action']); + } + + /** + * @param Message $message + * @param Document $project + * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump + * @return void + * @throws \Utopia\Database\Exception + * @throws Exception + */ + public function action(Message $message, Document $project, callable $getProjectDB, callable $getLogsDB, Database $dbForPlatform, callable $logError): void + { + $this->logError = $logError; + + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + if (empty($project->getAttribute('database'))) { + var_dump($payload); + return; + } + + $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); + } + + + protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void + { + Console::info('Begining count for: ' . $project->getId()); + + try { + /** @var \Utopia\Database\Database $dbForLogs */ + $dbForLogs = call_user_func($getLogsDB, $project); + /** @var \Utopia\Database\Database $dbForProject */ + $dbForProject = call_user_func($getProjectDB, $project); + + $region = $project->getAttribute('region'); + + $platforms = $dbForPlatform->count('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $webhooks = $dbForPlatform->count('webhooks', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $keys = $dbForPlatform->count('keys', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $databases = $dbForProject->count('databases'); + $buckets = $dbForProject->count('buckets'); + $users = $dbForProject->count('users'); + + $last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00'); + $usersActive = $dbForProject->count('users', [ + Query::greaterThanEqual('accessedAt', $last30Days) + ]); + $teams = $dbForProject->count('teams'); + $functions = $dbForProject->count('functions'); + $messages = $dbForProject->count('messages'); + $providers = $dbForProject->count('providers'); + $topics = $dbForProject->count('topics'); + + $metrics = [ + METRIC_DATABASES => $databases, + METRIC_BUCKETS => $buckets, + METRIC_USERS => $users, + METRIC_FUNCTIONS => $functions, + METRIC_TEAMS => $teams, + METRIC_MESSAGES => $messages, + METRIC_USERS_ACTIVE => $usersActive, + METRIC_WEBHOOKS => $webhooks, + METRIC_PLATFORMS => $platforms, + METRIC_PROVIDERS => $providers, + METRIC_TOPICS => $topics, + METRIC_KEYS => $keys, + ]; + + foreach ($metrics as $metric => $value) { + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $value); + } + + try { + $this->countForBuckets($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); + } + + try { + $this->countForDatabase($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]); + } + + try { + $this->countForFunctions($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]); + } + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); + } + + Console::info('End of count for: ' . $project->getId()); + } + + protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region) + { + $totalFiles = 0; + $totalStorage = 0; + $this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) { + $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); + + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $files); + + $storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual'); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $storage); + + $totalStorage += $storage; + $totalFiles += $files; + }); + + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES, $totalFiles); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES_STORAGE, $totalStorage); + } + + protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region) + { + $totalCollections = 0; + $totalDocuments = 0; + + $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) { + $collections = $dbForProject->count('database_' . $database->getInternalId()); + + $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $collections); + + $documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region); + + $totalDocuments += $documents; + $totalCollections += $collections; + }); + + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_COLLECTIONS, $totalCollections); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DOCUMENTS, $totalDocuments); + } + protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int + { + $databaseDocuments = 0; + $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) { + $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + + $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $documents); + + $databaseDocuments += $documents; + }); + + $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $databaseDocuments); + + return $databaseDocuments; + } + + protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) + { + $deploymentsStorage = $dbForProject->sum('deployments', 'size'); + $buildsStorage = $dbForProject->sum('builds', 'size'); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS_STORAGE, $buildsStorage); + + $deployments = $dbForProject->count('deployments'); + $builds = $dbForProject->count('builds'); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS, $deployments); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS, $builds); + + + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { + $functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]); + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage); + + $functionDeployments = $dbForProject->count('deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]); + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments); + + /** + * As deployments and builds have 1-1 relationship, + * the count for one should match the other + */ + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments); + + $functionBuildsStorage = 0; + + $this->foreachDocument($dbForProject, 'deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void { + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $functionBuildsStorage += $build->getAttribute('size', 0); + }); + + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); + }); + } + + protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value) + { + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : \date($format, \time()); + $id = \md5("{$time}_{$period}_{$metric}"); + $current = $dbForLogs->getDocument('usage', $id); + if ($current->isEmpty()) { + $dbForLogs->createDocument('usage', new Document([ + '$id' => $id, + 'metric' => $metric, + 'period' => $period, + 'region' => $region, + 'value' => $value, + ])); + Console::success('Usage logs created for metric: ' . $metric . ' period:'. $period); + } else { + $dbForLogs->updateDocument('usage', $id, $current->setAttribute('value', $value)); + Console::success('Usage logs updated for metric: ' . $metric . ' period:'. $period); + } + } + } +} From e78020d45a27e5ac1ed42625d249c3ebd0e421ef Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:06:57 +0000 Subject: [PATCH 18/57] stats resources evenet --- src/Appwrite/Event/Event.php | 3 ++ src/Appwrite/Event/StatsResources.php | 40 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 src/Appwrite/Event/StatsResources.php diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 5cd5f8e7d6..9c9244c056 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -27,6 +27,9 @@ class Event public const USAGE_QUEUE_NAME = 'v1-usage'; public const USAGE_CLASS_NAME = 'UsageV1'; + public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources'; + public const STATS_RESOURCES_CLASS_NAME = 'StatsResources'; + public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; diff --git a/src/Appwrite/Event/StatsResources.php b/src/Appwrite/Event/StatsResources.php new file mode 100644 index 0000000000..d08e1b2c8e --- /dev/null +++ b/src/Appwrite/Event/StatsResources.php @@ -0,0 +1,40 @@ +setQueue(Event::STATS_RESOURCES_QUEUE_NAME) + ->setClass(Event::STATS_RESOURCES_CLASS_NAME); + } + + /** + * Prepare the payload for the usage event. + * + * @return array + */ + protected function preparePayload(): array + { + return [ + 'project' => $this->project + ]; + } + + /** + * Sends metrics to the usage worker. + * + * @return string|bool + */ + public function trigger(): string|bool + { + parent::trigger(); + return true; + } +} From a05d77437058a540ef5a8e8fdf2607bfbb41073e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:07:44 +0000 Subject: [PATCH 19/57] queue resources to cli --- app/cli.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/cli.php b/app/cli.php index 91fef8dc55..a895275f76 100644 --- a/app/cli.php +++ b/app/cli.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/init.php'; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; +use Appwrite\Event\StatsResources; use Appwrite\Event\Usage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; @@ -209,6 +210,9 @@ CLI::setResource('queueForCertificates', function (Connection $queue) { CLI::setResource('queueForUsage', function (Connection $queue) { return new Usage($queue); }, ['queue']); +CLI::setResource('queueForStatsResources', function (Connection $queue) { + return new StatsResources($queue); +}, ['queue']); CLI::setResource('logError', function (Registry $register) { return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); From c2ce2730654df51cce142fda371bac0e8d261301 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:10:35 +0000 Subject: [PATCH 20/57] update stats resources task to use new queue --- .../Platform/Tasks/StatsResources.php | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index 5f077bbe9f..3445a009ec 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Tasks; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsResources as EventStatsResources; use Appwrite\Platform\Action; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -11,8 +11,6 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\System\System; -const INFINITY_PERIOD = '_inf_'; - /** * Usage count * @@ -35,13 +33,6 @@ class StatsResources extends Action */ protected Database $dbForPlatform; - /** - * Queue for usage - * - * @var Usage - */ - protected Usage $queue; - public static function getName() { return 'stats-resources'; @@ -53,23 +44,22 @@ class StatsResources extends Action ->desc('Schedules projects for usage count') ->inject('dbForPlatform') ->inject('logError') - ->inject('queueForUsage') - ->callback(fn ($dbForPlatform, $logError, $queueForUsage) => $this->action($dbForPlatform, $logError, $queueForUsage)); + ->inject('queueForStatsResources') + ->callback([$this, 'action']); } - public function action(Database $dbForPlatform, callable $logError, Usage $queueForUsage): void + public function action(Database $dbForPlatform, callable $logError, EventStatsResources $queueForStatsResources): void { $this->logError = $logError; $this->dbForPlatform = $dbForPlatform; - $this->queue = $queueForUsage; Console::title("Usage count V1"); Console::success('Usage count: Started'); $interval = (int) System::getEnv('_APP_USAGE_COUNT_INTERVAL', '3600'); - Console::loop(function () use ($queueForUsage) { - $this->enqueueProjects($queueForUsage); + Console::loop(function () use ($queueForStatsResources) { + $this->enqueueProjects($queueForStatsResources); }, $interval); Console::log("Usage count: Exited"); @@ -78,22 +68,23 @@ class StatsResources extends Action /** * Enqueue projects for counting * @param Database $dbForPlatform - * @param Usage $queue + * @param EventStatsResources $queue * @return void */ - protected function enqueueProjects(Usage $queue): void + protected function enqueueProjects(EventStatsResources $queue): void { Authorization::disable(); Authorization::setDefaultStatus(false); $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); - // Foreach Team + /** + * For each project that were accessed in last 24 hours + */ $this->foreachDocument($this->dbForPlatform, 'projects', [ Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours)) ], function ($project) use ($queue) { $queue ->setProject($project) - ->setType(Usage::TYPE_USAGE_COUNT) ->trigger(); Console::success('project: ' . $project->getId() . '(' . $project->getInternalId() . ')' . ' queued'); }); From 3fde465dfa541b1f6ce4b5d27b43a4b3ac6aad4b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:15:03 +0000 Subject: [PATCH 21/57] Update stats resources task container --- docker-compose.yml | 2 +- src/Appwrite/Platform/Tasks/StatsResources.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d6345abddb..4ea4336485 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -752,7 +752,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - - _APP_USAGE_COUNT_INTERVAL + - _APP_STATS_RESOURCES_INTERVAL appwrite-worker-stats-resources: entrypoint: worker-stats-resources diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index 3445a009ec..b2963ca87b 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -53,16 +53,16 @@ class StatsResources extends Action $this->logError = $logError; $this->dbForPlatform = $dbForPlatform; - Console::title("Usage count V1"); + Console::title("Stats resources V1"); - Console::success('Usage count: Started'); + Console::success('Stats resources: started'); - $interval = (int) System::getEnv('_APP_USAGE_COUNT_INTERVAL', '3600'); + $interval = (int) System::getEnv('_APP_STATS_RESOURCES_INTERVAL', '3600'); Console::loop(function () use ($queueForStatsResources) { $this->enqueueProjects($queueForStatsResources); }, $interval); - Console::log("Usage count: Exited"); + Console::log("Stats resources: exited"); } /** From 469dbf0a3155a254be20c3dca3a1642c98b0c209 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:33:42 +0000 Subject: [PATCH 22/57] reset usage --- src/Appwrite/Event/Usage.php | 39 +--- src/Appwrite/Platform/Workers/Usage.php | 280 ++---------------------- 2 files changed, 26 insertions(+), 293 deletions(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 7d630bf122..55f6bd6311 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -3,18 +3,14 @@ namespace Appwrite\Event; use Utopia\Database\Document; +use Utopia\Queue\Client; use Utopia\Queue\Connection; class Usage extends Event { - public const TYPE_USAGE_DUMP = 'usage-dump'; - public const TYPE_USAGE_COUNT = 'stats-resources'; - protected array $metrics = []; protected array $reduce = []; - protected string $type = self::TYPE_USAGE_DUMP; - public function __construct(protected Connection $connection) { parent::__construct($connection); @@ -24,12 +20,6 @@ class Usage extends Event ->setClass(Event::USAGE_CLASS_NAME); } - public function setType(string $type): self - { - $this->type = $type; - return $this; - } - /** * Add reduce. * @@ -52,7 +42,6 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { - $this->metrics[] = [ 'key' => $key, 'value' => $value, @@ -61,21 +50,6 @@ class Usage extends Event return $this; } - /** - * Prepare the payload for the usage event. - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'project' => $this->project, - 'reduce' => $this->reduce, - 'metrics' => $this->metrics, - 'type' => $this->type, - ]; - } - /** * Sends metrics to the usage worker. * @@ -83,8 +57,11 @@ class Usage extends Event */ public function trigger(): string|bool { - parent::trigger(); - $this->metrics = []; - return true; + $client = new Client($this->queue, $this->connection); + return $client->enqueue([ + 'project' => $this->getProject(), + 'reduce' => $this->reduce, + 'metrics' => $this->metrics, + ]); } -} +} \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 039dad939b..1b5abd4443 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -2,41 +2,22 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\Usage as UsageEvent; use Appwrite\Event\UsageDump; -use Appwrite\Platform\Action; use Exception; -use Throwable; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Query; +use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; class Usage extends Action { - /** - * Date format for different periods - */ - protected array $periods = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; - /** - * Log Error Callback - * - * @var callable - */ - protected mixed $logError; private array $stats = []; private int $lastTriggeredTime = 0; - private int $aggregationInterval = 20; private int $keys = 0; private const INFINITY_PERIOD = '_inf_'; - private const KEYS_THRESHOLD = 20000; + private const KEYS_THRESHOLD = 10000; public static function getName(): string { @@ -48,50 +29,37 @@ class Usage extends Action */ public function __construct() { - $this - ->desc('Usage worker') - ->inject('message') - ->inject('project') - ->inject('getProjectDB') - ->inject('queueForUsageDump') - ->inject('getLogsDB') - ->inject('dbForPlatform') - ->inject('logError') - ->callback([$this, 'action']); - $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); + $this + ->desc('Usage worker') + ->inject('message') + ->inject('getProjectDB') + ->inject('queueForUsageDump') + ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $getProjectDB, $queueForUsageDump); + }); + $this->lastTriggeredTime = time(); } /** * @param Message $message - * @param Document $project * @param callable $getProjectDB * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump, callable $getLogsDB, Database $dbForPlatform, callable $logError): void + public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void { - $this->logError = $logError; - $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } + //Todo Figure out way to preserve keys when the container is being recreated @shimonewman - if (empty($project->getAttribute('database'))) { - var_dump($payload); - return; - } - - $type = $payload['type'] ?? UsageEvent::TYPE_USAGE_DUMP; - if ($type === UsageEvent::TYPE_USAGE_COUNT) { - $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); - return; - } - + $aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); + $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { if (empty($document)) { @@ -106,12 +74,7 @@ class Usage extends Action ); } - - $this->stats[$projectId]['project'] = [ - '$id' => $project->getId(), - '$internalId' => $project->getInternalId(), - 'database' => $project->getAttribute('database'), - ]; + $this->stats[$projectId]['project'] = $project; $this->stats[$projectId]['receivedAt'] = DateTime::now(); foreach ($payload['metrics'] ?? [] as $metric) { $this->keys++; @@ -126,7 +89,7 @@ class Usage extends Action // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || - (time() - $this->lastTriggeredTime > $this->aggregationInterval && $this->keys > 0) + (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); @@ -314,211 +277,4 @@ class Usage extends Action console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } - - - protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void - { - Console::info('Begining count for: ' . $project->getId()); - - try { - /** @var \Utopia\Database\Database $dbForLogs */ - $dbForLogs = call_user_func($getLogsDB, $project); - /** @var \Utopia\Database\Database $dbForProject */ - $dbForProject = call_user_func($getProjectDB, $project); - - $region = $project->getAttribute('region'); - - $platforms = $dbForPlatform->count('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ]); - $webhooks = $dbForPlatform->count('webhooks', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ]); - $keys = $dbForPlatform->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ]); - $databases = $dbForProject->count('databases'); - $buckets = $dbForProject->count('buckets'); - $users = $dbForProject->count('users'); - - $last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00'); - $usersActive = $dbForProject->count('users', [ - Query::greaterThanEqual('accessedAt', $last30Days) - ]); - $teams = $dbForProject->count('teams'); - $functions = $dbForProject->count('functions'); - $messages = $dbForProject->count('messages'); - $providers = $dbForProject->count('providers'); - $topics = $dbForProject->count('topics'); - - $metrics = [ - METRIC_DATABASES => $databases, - METRIC_BUCKETS => $buckets, - METRIC_USERS => $users, - METRIC_FUNCTIONS => $functions, - METRIC_TEAMS => $teams, - METRIC_MESSAGES => $messages, - METRIC_USERS_ACTIVE => $usersActive, - METRIC_WEBHOOKS => $webhooks, - METRIC_PLATFORMS => $platforms, - METRIC_PROVIDERS => $providers, - METRIC_TOPICS => $topics, - METRIC_KEYS => $keys, - ]; - - foreach ($metrics as $metric => $value) { - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $value); - } - - try { - $this->countForBuckets($dbForProject, $dbForLogs, $region); - } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); - } - - try { - $this->countForDatabase($dbForProject, $dbForLogs, $region); - } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]); - } - - try { - $this->countForFunctions($dbForProject, $dbForLogs, $region); - } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]); - } - } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); - } - - Console::info('End of count for: ' . $project->getId()); - } - - protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region) - { - $totalFiles = 0; - $totalStorage = 0; - $this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) { - $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); - - $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $files); - - $storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual'); - $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $storage); - - $totalStorage += $storage; - $totalFiles += $files; - }); - - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES, $totalFiles); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES_STORAGE, $totalStorage); - } - - protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region) - { - $totalCollections = 0; - $totalDocuments = 0; - - $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) { - $collections = $dbForProject->count('database_' . $database->getInternalId()); - - $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $collections); - - $documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region); - - $totalDocuments += $documents; - $totalCollections += $collections; - }); - - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_COLLECTIONS, $totalCollections); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DOCUMENTS, $totalDocuments); - } - protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int - { - $databaseDocuments = 0; - $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) { - $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - - $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $documents); - - $databaseDocuments += $documents; - }); - - $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $databaseDocuments); - - return $databaseDocuments; - } - - protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) - { - $deploymentsStorage = $dbForProject->sum('deployments', 'size'); - $buildsStorage = $dbForProject->sum('builds', 'size'); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS_STORAGE, $buildsStorage); - - $deployments = $dbForProject->count('deployments'); - $builds = $dbForProject->count('builds'); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS, $deployments); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS, $builds); - - - $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { - $functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [ - Query::equal('resourceInternalId', [$function->getInternalId()]), - Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), - ]); - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage); - - $functionDeployments = $dbForProject->count('deployments', [ - Query::equal('resourceInternalId', [$function->getInternalId()]), - Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), - ]); - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments); - - /** - * As deployments and builds have 1-1 relationship, - * the count for one should match the other - */ - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments); - - $functionBuildsStorage = 0; - - $this->foreachDocument($dbForProject, 'deployments', [ - Query::equal('resourceInternalId', [$function->getInternalId()]), - Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), - ], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - $functionBuildsStorage += $build->getAttribute('size', 0); - }); - - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); - }); - } - - protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value) - { - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : \date($format, \time()); - $id = \md5("{$time}_{$period}_{$metric}"); - $current = $dbForLogs->getDocument('usage', $id); - if ($current->isEmpty()) { - $dbForLogs->createDocument('usage', new Document([ - '$id' => $id, - 'metric' => $metric, - 'period' => $period, - 'region' => $region, - 'value' => $value, - ])); - Console::success('Usage logs created for metric: ' . $metric . ' period:'. $period); - } else { - $dbForLogs->updateDocument('usage', $id, $current->setAttribute('value', $value)); - Console::success('Usage logs updated for metric: ' . $metric . ' period:'. $period); - } - } - } -} +} \ No newline at end of file From 0da3f870f9a8bb12e67977ad1b5f1369f1cf1018 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:43:16 +0000 Subject: [PATCH 23/57] refactor: rename --- Dockerfile | 2 +- app/controllers/api/health.php | 10 +++++----- app/views/install/compose.phtml | 6 +++--- app/worker.php | 6 +++--- bin/worker-stats-usage-dump | 3 +++ bin/worker-usage-dump | 3 --- docker-compose.yml | 6 +++--- src/Appwrite/Event/Event.php | 4 ++-- .../Event/{UsageDump.php => StatsUsageDump.php} | 6 +++--- src/Appwrite/Platform/Services/Workers.php | 4 ++-- src/Appwrite/Platform/Workers/StatsResources.php | 1 - .../Workers/{UsageDump.php => StatsUsageDump.php} | 4 ++-- src/Appwrite/Platform/Workers/Usage.php | 14 +++++++------- .../e2e/Services/Health/HealthCustomServerTest.php | 6 +++--- 14 files changed, 37 insertions(+), 38 deletions(-) create mode 100644 bin/worker-stats-usage-dump delete mode 100644 bin/worker-usage-dump rename src/Appwrite/Event/{UsageDump.php => StatsUsageDump.php} (81%) rename src/Appwrite/Platform/Workers/{UsageDump.php => StatsUsageDump.php} (99%) diff --git a/Dockerfile b/Dockerfile index 1d50684535..f764e16055 100755 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-usage && \ - chmod +x /usr/local/bin/worker-usage-dump && \ + chmod +x /usr/local/bin/worker-stats-usage-dump && \ chmod +x /usr/local/bin/stats-resources && \ chmod +x /usr/local/bin/worker-stats-resources diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 1db4713311..ab3551fcd9 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -798,15 +798,15 @@ App::get('/v1/health/queue/usage') $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }); -App::get('/v1/health/queue/usage-dump') +App::get('/v1/health/queue/stats-usage-dump') ->desc('Get usage dump queue') ->groups(['api', 'health']) ->label('scope', 'health.read') ->label('sdk', new Method( auth: [AuthType::KEY], namespace: 'health', - name: 'getQueueUsageDump', - description: '/docs/references/health/get-queue-usage-dump.md', + name: 'getQueueStatsUsageDump', + description: '/docs/references/health/get-queue-stats-usage-dump.md', responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, @@ -821,7 +821,7 @@ App::get('/v1/health/queue/usage-dump') ->action(function (int|string $threshold, Connection $queue, Response $response) { $threshold = \intval($threshold); - $client = new Client(Event::USAGE_DUMP_QUEUE_NAME, $queue); + $client = new Client(Event::STATS_USAGE_DUMP_QUEUE_NAME, $queue); $size = $client->getQueueSize(); if ($size >= $threshold) { @@ -996,7 +996,7 @@ App::get('/v1/health/queue/failed/:name') Event::MAILS_QUEUE_NAME, Event::FUNCTIONS_QUEUE_NAME, Event::USAGE_QUEUE_NAME, - Event::USAGE_DUMP_QUEUE_NAME, + Event::STATS_USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_QUEUE_NAME, Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index ad6d883c4c..ff6da07c78 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -677,11 +677,11 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-worker-usage-dump: + appwrite-worker-stats-usage-dump: image: /: - entrypoint: worker-usage-dump + entrypoint: worker-stats-usage-dump <<: *x-logging - container_name: appwrite-worker-usage-dump + container_name: appwrite-worker-stats-usage-dump restart: unless-stopped networks: - appwrite diff --git a/app/worker.php b/app/worker.php index 81b1367873..9658202a41 100644 --- a/app/worker.php +++ b/app/worker.php @@ -14,7 +14,7 @@ use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; use Appwrite\Event\Usage; -use Appwrite\Event\UsageDump; +use Appwrite\Event\StatsUsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; @@ -261,8 +261,8 @@ Server::setResource('queueForUsage', function (Connection $queue) { return new Usage($queue); }, ['queue']); -Server::setResource('queueForUsageDump', function (Connection $queue) { - return new UsageDump($queue); +Server::setResource('queueForStatsUsageDump', function (Connection $queue) { + return new StatsUsageDump($queue); }, ['queue']); Server::setResource('queue', function (Group $pools) { diff --git a/bin/worker-stats-usage-dump b/bin/worker-stats-usage-dump new file mode 100644 index 0000000000..98e3c2cac7 --- /dev/null +++ b/bin/worker-stats-usage-dump @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php stats-usage-dump $@ \ No newline at end of file diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump deleted file mode 100644 index 43ca87fcb3..0000000000 --- a/bin/worker-usage-dump +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4ea4336485..7d04871061 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -816,10 +816,10 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-usage-dump: - entrypoint: worker-usage-dump + appwrite-worker-stats-usage-dump: + entrypoint: worker-stats-usage-dump <<: *x-logging - container_name: appwrite-worker-usage-dump + container_name: appwrite-worker-stats-usage-dump image: appwrite-dev networks: - appwrite diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 9c9244c056..a1c5abd1d8 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -30,8 +30,8 @@ class Event public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources'; public const STATS_RESOURCES_CLASS_NAME = 'StatsResources'; - public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; - public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; + public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump'; + public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1'; public const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; public const WEBHOOK_CLASS_NAME = 'WebhooksV1'; diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/StatsUsageDump.php similarity index 81% rename from src/Appwrite/Event/UsageDump.php rename to src/Appwrite/Event/StatsUsageDump.php index 6f44de4eda..3cd38eca92 100644 --- a/src/Appwrite/Event/UsageDump.php +++ b/src/Appwrite/Event/StatsUsageDump.php @@ -4,7 +4,7 @@ namespace Appwrite\Event; use Utopia\Queue\Connection; -class UsageDump extends Event +class StatsUsageDump extends Event { protected array $stats; @@ -13,8 +13,8 @@ class UsageDump extends Event parent::__construct($connection); $this - ->setQueue(Event::USAGE_DUMP_QUEUE_NAME) - ->setClass(Event::USAGE_DUMP_CLASS_NAME); + ->setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME) + ->setClass(Event::STATS_USAGE_DUMP_CLASS_NAME); } /** diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 150c8cfd23..c1198231f9 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -13,7 +13,7 @@ use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\Usage; -use Appwrite\Platform\Workers\UsageDump; +use Appwrite\Platform\Workers\StatsUsageDump; use Appwrite\Platform\Workers\Webhooks; use Utopia\Platform\Service; @@ -32,7 +32,7 @@ class Workers extends Service ->addAction(Mails::getName(), new Mails()) ->addAction(Messaging::getName(), new Messaging()) ->addAction(Webhooks::getName(), new Webhooks()) - ->addAction(UsageDump::getName(), new UsageDump()) + ->addAction(StatsUsageDump::getName(), new StatsUsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) ->addAction(StatsResources::getName(), new StatsResources()) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index e4ba59c464..4f6855a6a1 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -47,7 +47,6 @@ class StatsResources extends Action * @param Message $message * @param Document $project * @param callable $getProjectDB - * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php similarity index 99% rename from src/Appwrite/Platform/Workers/UsageDump.php rename to src/Appwrite/Platform/Workers/StatsUsageDump.php index 0e7103b8c2..fa31691a18 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -16,7 +16,7 @@ const METRIC_COLLECTION_LEVEL_STORAGE = 4; const METRIC_DATABASE_LEVEL_STORAGE = 3; const METRIC_PROJECT_LEVEL_STORAGE = 2; -class UsageDump extends Action +class StatsUsageDump extends Action { protected array $stats = []; @@ -72,7 +72,7 @@ class UsageDump extends Action public static function getName(): string { - return 'usage-dump'; + return 'stats-usage-dump'; } /** diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 1b5abd4443..176a9fea2f 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\UsageDump; +use Appwrite\Event\StatsUsageDump; use Exception; use Utopia\CLI\Console; use Utopia\Database\DateTime; @@ -34,9 +34,9 @@ class Usage extends Action ->desc('Usage worker') ->inject('message') ->inject('getProjectDB') - ->inject('queueForUsageDump') - ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { - $this->action($message, $getProjectDB, $queueForUsageDump); + ->inject('queueForStatsUsageDump') + ->callback(function (Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump) { + $this->action($message, $getProjectDB, $queueForStatsUsageDump); }); $this->lastTriggeredTime = time(); @@ -45,12 +45,12 @@ class Usage extends Action /** * @param Message $message * @param callable $getProjectDB - * @param UsageDump $queueForUsageDump + * @param StatsUsageDump $queueForStatsUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void + public function action(Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { @@ -93,7 +93,7 @@ class Usage extends Action ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); - $queueForUsageDump + $queueForStatsUsageDump ->setStats($this->stats) ->trigger(); diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 9d6a04abe6..b05351dce5 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -536,12 +536,12 @@ class HealthCustomServerTest extends Scope $this->assertEquals(503, $response['headers']['status-code']); } - public function testUsageDumpSuccess() + public function testStatsUsageDumpSuccess() { /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -553,7 +553,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump?threshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); From cb31e7954ee7b1b1ddece154d2860ffb730e7230 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:53:53 +0000 Subject: [PATCH 24/57] refactor rename --- CONTRIBUTING.md | 12 ++--- Dockerfile | 2 +- app/cli.php | 6 +-- app/controllers/api/account.php | 26 +++++----- app/controllers/api/databases.php | 50 +++++++++---------- app/controllers/api/functions.php | 8 +-- app/controllers/api/storage.php | 8 +-- app/controllers/api/teams.php | 10 ++-- app/controllers/general.php | 38 +++++++------- app/controllers/shared/api.php | 50 +++++++++---------- app/init.php | 6 +-- app/views/install/compose.phtml | 6 +-- app/worker.php | 6 +-- bin/worker-stats-usage | 3 ++ bin/worker-usage | 3 -- docker-compose.yml | 6 +-- .../Event/{Usage.php => StatsUsage.php} | 4 +- src/Appwrite/Platform/Services/Workers.php | 4 +- src/Appwrite/Platform/Workers/Builds.php | 20 ++++---- src/Appwrite/Platform/Workers/Functions.php | 20 ++++---- src/Appwrite/Platform/Workers/Messaging.php | 24 ++++----- .../Workers/{Usage.php => StatsUsage.php} | 6 +-- src/Appwrite/Platform/Workers/Webhooks.php | 18 +++---- tests/resources/docker/docker-compose.yml | 6 +-- 24 files changed, 171 insertions(+), 171 deletions(-) create mode 100644 bin/worker-stats-usage delete mode 100644 bin/worker-usage rename src/Appwrite/Event/{Usage.php => StatsUsage.php} (97%) rename src/Appwrite/Platform/Workers/{Usage.php => StatsUsage.php} (99%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e89aa369cf..d53e3786da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -367,7 +367,7 @@ In file `app/controllers/shared/api.php` On the database listener, add to an exi ```php case $document->getCollection() === 'teams': - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_TEAMS, $value); // per project break; ``` @@ -379,10 +379,10 @@ In that case you need also to handle children removal using addReduce() method c ```php case $document->getCollection() === 'buckets': //buckets - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage + $queueForStatsUsage ->addReduce($document); } break; @@ -428,16 +428,16 @@ public function __construct() ->inject('dbForProject') ->inject('queueForFunctions') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('log') - ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log)); + ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log)); } ``` and then trigger the queue with the new metric like so: ```php -$queueForUsage +$queueForStatsUsage ->addMetric(METRIC_BUILDS, 1) ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) diff --git a/Dockerfile b/Dockerfile index f764e16055..01ad5d3fba 100755 --- a/Dockerfile +++ b/Dockerfile @@ -85,7 +85,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ - chmod +x /usr/local/bin/worker-usage && \ + chmod +x /usr/local/bin/worker-stats-usage && \ chmod +x /usr/local/bin/worker-stats-usage-dump && \ chmod +x /usr/local/bin/stats-resources && \ chmod +x /usr/local/bin/worker-stats-resources diff --git a/app/cli.php b/app/cli.php index a895275f76..df6c6dff51 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,7 +6,7 @@ use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; use Appwrite\Event\StatsResources; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; use Utopia\Cache\Adapter\Sharding; @@ -207,8 +207,8 @@ CLI::setResource('queueForDeletes', function (Connection $queue) { CLI::setResource('queueForCertificates', function (Connection $queue) { return new Certificate($queue); }, ['queue']); -CLI::setResource('queueForUsage', function (Connection $queue) { - return new Usage($queue); +CLI::setResource('queueForStatsUsage', function (Connection $queue) { + return new StatsUsage($queue); }, ['queue']); CLI::setResource('queueForStatsResources', function (Connection $queue) { return new StatsResources($queue); diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index bd9562110f..21a04800b5 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -17,7 +17,7 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Hooks\Hooks; use Appwrite\Network\Validator\Email; @@ -2432,9 +2432,9 @@ App::post('/v1/account/tokens/phone') ->inject('queueForMessaging') ->inject('locale') ->inject('timelimit') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('plan') - ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, Usage $queueForUsage, array $plan) { + ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -2583,11 +2583,11 @@ App::post('/v1/account/tokens/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForUsage + $queueForStatsUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) ->setProject($project) ->trigger(); @@ -3678,9 +3678,9 @@ App::post('/v1/account/verification/phone') ->inject('project') ->inject('locale') ->inject('timelimit') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('plan') - ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, Usage $queueForUsage, array $plan) { + ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -3775,11 +3775,11 @@ App::post('/v1/account/verification/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForUsage + $queueForStatsUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) ->setProject($project) ->trigger(); @@ -4310,9 +4310,9 @@ App::post('/v1/account/mfa/challenge') ->inject('queueForMessaging') ->inject('queueForMails') ->inject('timelimit') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('plan') - ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, Usage $queueForUsage, array $plan) { + ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) { $expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM); $code = Auth::codeGenerator(); @@ -4383,11 +4383,11 @@ App::post('/v1/account/mfa/challenge') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForUsage + $queueForStatsUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) ->setProject($project) ->trigger(); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index df46c1890b..98ba59106d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -4,7 +4,7 @@ use Appwrite\Auth\Auth; use Appwrite\Detector\Detector; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Event; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email; use Appwrite\SDK\AuthType; @@ -476,8 +476,8 @@ App::post('/v1/databases') ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForUsage') - ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $databaseId = $databaseId == 'unique()' ? ID::unique() : $databaseId; @@ -527,7 +527,7 @@ App::post('/v1/databases') } $queueForEvents->setParam('databaseId', $database->getId()); - $queueForUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database + $queueForStatsUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -797,8 +797,8 @@ App::delete('/v1/databases/:databaseId') ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForUsage') - ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -821,7 +821,7 @@ App::delete('/v1/databases/:databaseId') ->setParam('databaseId', $database->getId()) ->setPayload($response->output($database, Response::MODEL_DATABASE)); - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation $response->noContent(); @@ -2618,8 +2618,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForUsage') - ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $db = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -2716,7 +2716,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->setContext('database', $db) ->setPayload($response->output($attribute, $model)); - $queueForUsage + $queueForStatsUsage ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->noContent(); @@ -3134,9 +3134,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('mode') - ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode) { + ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, string $mode) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3339,7 +3339,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3392,8 +3392,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->inject('queueForUsage') - ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3505,7 +3505,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; @@ -3571,8 +3571,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('response') ->inject('dbForProject') ->inject('mode') - ->inject('queueForUsage') - ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, StatsUsage $queueForStatsUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3648,7 +3648,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; @@ -3805,8 +3805,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('dbForProject') ->inject('queueForEvents') ->inject('mode') - ->inject('queueForUsage') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, StatsUsage $queueForStatsUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3946,7 +3946,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; @@ -4058,9 +4058,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -4129,7 +4129,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 3d9240d494..14255ef7a4 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -6,7 +6,7 @@ use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Event\Validator\FunctionEvent; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; @@ -1900,10 +1900,10 @@ App::post('/v1/functions/:functionId/executions') ->inject('dbForPlatform') ->inject('user') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('queueForFunctions') ->inject('geodb') - ->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb) { $async = \strval($async) === 'true' || \strval($async) === '1'; if (!$async && !is_null($scheduledAt)) { @@ -2230,7 +2230,7 @@ App::post('/v1/functions/:functionId/executions') throw $th; } } finally { - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index b6a07c356d..a5ef287dcd 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -6,7 +6,7 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; @@ -941,8 +941,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->inject('queueForUsage') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1070,7 +1070,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) ; diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index e525086ebf..b2b897f88a 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -8,7 +8,7 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email; use Appwrite\Platform\Workers\Deletes; @@ -466,9 +466,9 @@ App::post('/v1/teams/:teamId/memberships') ->inject('queueForMessaging') ->inject('queueForEvents') ->inject('timelimit') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('plan') - ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, Usage $queueForUsage, array $plan) { + ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -759,11 +759,11 @@ App::post('/v1/teams/:teamId/memberships') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForUsage + $queueForStatsUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) ->setProject($project) ->trigger(); diff --git a/app/controllers/general.php b/app/controllers/general.php index 7e691d033f..1d56e79b74 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -7,7 +7,7 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Network\Validator\Origin; use Appwrite\SDK\AuthType; @@ -50,7 +50,7 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) +function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); @@ -434,7 +434,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) @@ -499,13 +499,13 @@ App::init() ->inject('localeCodes') ->inject('clients') ->inject('geodb') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('queueForEvents') ->inject('queueForCertificates') ->inject('queueForFunctions') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ @@ -513,7 +513,7 @@ App::init() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -732,12 +732,12 @@ App::options() ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ @@ -745,7 +745,7 @@ App::options() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -770,8 +770,8 @@ App::error() ->inject('project') ->inject('logger') ->inject('log') - ->inject('queueForUsage') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { + ->inject('queueForStatsUsage') + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, StatsUsage $queueForStatsUsage) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); @@ -882,13 +882,13 @@ App::error() $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); } - $queueForUsage + $queueForStatsUsage ->setProject($project) ->trigger(); } @@ -1040,12 +1040,12 @@ App::get('/robots.txt') ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -1053,7 +1053,7 @@ App::get('/robots.txt') $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); + router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); @@ -1068,12 +1068,12 @@ App::get('/humans.txt') ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -1081,7 +1081,7 @@ App::get('/humans.txt') $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); + router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e2845521f8..0b9456ac78 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -12,7 +12,7 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Messaging; use Appwrite\Event\Realtime; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; @@ -90,7 +90,7 @@ $eventDatabaseListener = function (Document $project, Document $document, Respon } }; -$usageDatabaseListener = function (string $event, Document $document, Usage $queueForUsage) { +$usageDatabaseListener = function (string $event, Document $document, StatsUsage $queueForStatsUsage) { $value = 1; if ($event === Database::EVENT_DOCUMENT_DELETE) { $value = -1; @@ -98,40 +98,40 @@ $usageDatabaseListener = function (string $event, Document $document, Usage $que switch (true) { case $document->getCollection() === 'teams': - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_TEAMS, $value); // per project break; case $document->getCollection() === 'users': - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_USERS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage + $queueForStatsUsage ->addReduce($document); } break; case $document->getCollection() === 'sessions': // sessions - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_SESSIONS, $value); //per project break; case $document->getCollection() === 'databases': // databases - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DATABASES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage + $queueForStatsUsage ->addReduce($document); } break; case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_COLLECTIONS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) ; if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage + $queueForStatsUsage ->addReduce($document); } break; @@ -139,39 +139,39 @@ $usageDatabaseListener = function (string $event, Document $document, Usage $que $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; $collectionInternalId = $parts[3] ?? 0; - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DOCUMENTS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection break; case $document->getCollection() === 'buckets': //buckets - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage + $queueForStatsUsage ->addReduce($document); } break; case str_starts_with($document->getCollection(), 'bucket_'): // files $parts = explode('_', $document->getCollection()); $bucketInternalId = $parts[1]; - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_FILES, $value) // per project ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket break; case $document->getCollection() === 'functions': - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_FUNCTIONS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForUsage + $queueForStatsUsage ->addReduce($document); } break; case $document->getCollection() === 'deployments': - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_DEPLOYMENTS, $value) // per project ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value) // per function @@ -436,11 +436,11 @@ App::init() ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('dbForProject') ->inject('timelimit') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -550,8 +550,8 @@ App::init() $queueForRealtime = new Realtime(); $dbForProject - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( $project, $document, @@ -688,7 +688,7 @@ App::shutdown() ->inject('user') ->inject('queueForEvents') ->inject('queueForAudits') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') @@ -697,7 +697,7 @@ App::shutdown() ->inject('queueForWebhooks') ->inject('queueForRealtime') ->inject('dbForProject') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -854,13 +854,13 @@ App::shutdown() $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); } - $queueForUsage + $queueForStatsUsage ->setProject($project) ->trigger(); } diff --git a/app/init.php b/app/init.php index 8a376823ce..228625b25b 100644 --- a/app/init.php +++ b/app/init.php @@ -32,7 +32,7 @@ use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; use Appwrite\Event\Realtime; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Functions\Specification; @@ -1188,8 +1188,8 @@ App::setResource('queueForAudits', function (Connection $queue) { App::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); -App::setResource('queueForUsage', function (Connection $queue) { - return new Usage($queue); +App::setResource('queueForStatsUsage', function (Connection $queue) { + return new StatsUsage($queue); }, ['queue']); App::setResource('queueForCertificates', function (Connection $queue) { return new Certificate($queue); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index ff6da07c78..c69ab60015 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -649,10 +649,10 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - appwrite-worker-usage: + appwrite-worker-stats-usage: image: /: - entrypoint: worker-usage - container_name: appwrite-worker-usage + entrypoint: worker-stats-usage + container_name: appwrite-worker-stats-usage <<: *x-logging restart: unless-stopped networks: diff --git a/app/worker.php b/app/worker.php index 9658202a41..415d6a6f27 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,7 +13,7 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; @@ -257,8 +257,8 @@ Server::setResource('timelimit', function (\Redis $redis) { Server::setResource('log', fn () => new Log()); -Server::setResource('queueForUsage', function (Connection $queue) { - return new Usage($queue); +Server::setResource('queueForStatsUsage', function (Connection $queue) { + return new StatsUsage($queue); }, ['queue']); Server::setResource('queueForStatsUsageDump', function (Connection $queue) { diff --git a/bin/worker-stats-usage b/bin/worker-stats-usage new file mode 100644 index 0000000000..2c267d805e --- /dev/null +++ b/bin/worker-stats-usage @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php stats-usage $@ \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index e39ce8477c..0000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php usage $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7d04871061..7b05a48cc6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -785,10 +785,10 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-usage: - entrypoint: worker-usage + appwrite-worker-stats-usage: + entrypoint: worker-stats-usage <<: *x-logging - container_name: appwrite-worker-usage + container_name: appwrite-worker-stats-usage image: appwrite-dev networks: - appwrite diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/StatsUsage.php similarity index 97% rename from src/Appwrite/Event/Usage.php rename to src/Appwrite/Event/StatsUsage.php index 55f6bd6311..7f54370fca 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -6,7 +6,7 @@ use Utopia\Database\Document; use Utopia\Queue\Client; use Utopia\Queue\Connection; -class Usage extends Event +class StatsUsage extends Event { protected array $metrics = []; protected array $reduce = []; @@ -64,4 +64,4 @@ class Usage extends Event 'metrics' => $this->metrics, ]); } -} \ No newline at end of file +} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index c1198231f9..4f4095aca4 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -12,7 +12,7 @@ use Appwrite\Platform\Workers\Mails; use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; use Appwrite\Platform\Workers\StatsResources; -use Appwrite\Platform\Workers\Usage; +use Appwrite\Platform\Workers\StatsUsage; use Appwrite\Platform\Workers\StatsUsageDump; use Appwrite\Platform\Workers\Webhooks; use Utopia\Platform\Service; @@ -33,7 +33,7 @@ class Workers extends Service ->addAction(Messaging::getName(), new Messaging()) ->addAction(Webhooks::getName(), new Webhooks()) ->addAction(StatsUsageDump::getName(), new StatsUsageDump()) - ->addAction(Usage::getName(), new Usage()) + ->addAction(StatsUsage::getName(), new StatsUsage()) ->addAction(Migrations::getName(), new Migrations()) ->addAction(StatsResources::getName(), new StatsResources()) ; diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 4f5d6eb694..f6e06ac0e5 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; @@ -50,7 +50,7 @@ class Builds extends Action ->inject('dbForPlatform') ->inject('queueForEvents') ->inject('queueForFunctions') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('cache') ->inject('dbForProject') ->inject('deviceForFunctions') @@ -64,7 +64,7 @@ class Builds extends Action * @param Database $dbForPlatform * @param Event $queueForEvents * @param Func $queueForFunctions - * @param Usage $queueForUsage + * @param StatsUsage $queueForStatsUsage * @param Cache $cache * @param Database $dbForProject * @param Device $deviceForFunctions @@ -72,7 +72,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -93,7 +93,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); + $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); break; default: @@ -105,7 +105,7 @@ class Builds extends Action * @param Device $deviceForFunctions * @param Func $queueForFunctions * @param Event $queueForEvents - * @param Usage $queueForUsage + * @param StatsUsage $queueForStatsUsage * @param Database $dbForPlatform * @param Database $dbForProject * @param GitHub $github @@ -118,7 +118,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -706,20 +706,20 @@ class Builds extends Action /** Trigger usage queue */ if ($build->getAttribute('status') === 'ready') { - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); } elseif ($build->getAttribute('status') === 'failed') { - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_BUILDS_FAILED, 1) // per project ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000); } - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 72a3334f2f..0a7c39c02f 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Execution; use Exception; @@ -46,13 +46,13 @@ class Functions extends Action ->inject('dbForProject') ->inject('queueForFunctions') ->inject('queueForEvents') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('log') ->inject('isResourceBlocked') - ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log, $isResourceBlocked)); + ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); } - public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked): void + public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void { $payload = $message->getPayload() ?? []; @@ -137,7 +137,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - queueForUsage: $queueForUsage, + queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, function: $function, @@ -177,7 +177,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - queueForUsage: $queueForUsage, + queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, function: $function, @@ -199,7 +199,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - queueForUsage: $queueForUsage, + queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, function: $function, @@ -284,7 +284,7 @@ class Functions extends Action * @param Log $log * @param Database $dbForProject * @param Func $queueForFunctions - * @param Usage $queueForUsage + * @param StatsUsage $queueForStatsUsage * @param Event $queueForEvents * @param Document $project * @param Document $function @@ -308,7 +308,7 @@ class Functions extends Action Log $log, Database $dbForProject, Func $queueForFunctions, - Usage $queueForUsage, + StatsUsage $queueForStatsUsage, Event $queueForEvents, Document $project, Document $function, @@ -552,7 +552,7 @@ class Functions extends Action $errorCode = $th->getCode(); } finally { /** Trigger usage queue */ - $queueForUsage + $queueForStatsUsage ->setProject($project) ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7837957c5d..385cf31e78 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Messaging\Status as MessageStatus; use Swoole\Runtime; use Utopia\CLI\Console; @@ -63,8 +63,8 @@ class Messaging extends Action ->inject('log') ->inject('dbForProject') ->inject('deviceForFiles') - ->inject('queueForUsage') - ->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForUsage)); + ->inject('queueForStatsUsage') + ->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, StatsUsage $queueForStatsUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForStatsUsage)); } /** @@ -73,7 +73,7 @@ class Messaging extends Action * @param Log $log * @param Database $dbForProject * @param Device $deviceForFiles - * @param Usage $queueForUsage + * @param StatsUsage $queueForStatsUsage * @return void * @throws \Exception */ @@ -83,7 +83,7 @@ class Messaging extends Action Log $log, Database $dbForProject, Device $deviceForFiles, - Usage $queueForUsage + StatsUsage $queueForStatsUsage ): void { Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP); $payload = $message->getPayload() ?? []; @@ -104,7 +104,7 @@ class Messaging extends Action case MESSAGE_SEND_TYPE_EXTERNAL: $message = $dbForProject->getDocument('messages', $payload['messageId']); - $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForUsage); + $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForStatsUsage); break; default: throw new \Exception('Unknown message type: ' . $type); @@ -116,7 +116,7 @@ class Messaging extends Action Document $message, Device $deviceForFiles, Document $project, - Usage $queueForUsage + StatsUsage $queueForStatsUsage ): void { $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); @@ -222,8 +222,8 @@ class Messaging extends Action /** * @var array $results */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) { + $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { if (\array_key_exists($providerId, $providers)) { $provider = $providers[$providerId]; } else { @@ -250,8 +250,8 @@ class Messaging extends Action $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) { - return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) { + return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { $deliveredTotal = 0; $deliveryErrors = []; $messageData = clone $message; @@ -291,7 +291,7 @@ class Messaging extends Action $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); } finally { $errorTotal = \count($deliveryErrors); - $queueForUsage + $queueForStatsUsage ->setProject($project) ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/StatsUsage.php similarity index 99% rename from src/Appwrite/Platform/Workers/Usage.php rename to src/Appwrite/Platform/Workers/StatsUsage.php index 176a9fea2f..7ee4cb9890 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -11,7 +11,7 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; -class Usage extends Action +class StatsUsage extends Action { private array $stats = []; private int $lastTriggeredTime = 0; @@ -21,7 +21,7 @@ class Usage extends Action public static function getName(): string { - return 'usage'; + return 'stats-usage'; } /** @@ -277,4 +277,4 @@ class Usage extends Action console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } -} \ No newline at end of file +} diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index a76e4f17b0..c903dafdae 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Mail; -use Appwrite\Event\Usage; +use Appwrite\Event\StatsUsage; use Appwrite\Template\Template; use Exception; use Utopia\Database\Database; @@ -35,9 +35,9 @@ class Webhooks extends Action ->inject('project') ->inject('dbForPlatform') ->inject('queueForMails') - ->inject('queueForUsage') + ->inject('queueForStatsUsage') ->inject('log') - ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForUsage, $log)); + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage, $log)); } /** @@ -49,7 +49,7 @@ class Webhooks extends Action * @return void * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log): void { $this->errors = []; $payload = $message->getPayload() ?? []; @@ -66,7 +66,7 @@ class Webhooks extends Action foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { - $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForUsage); + $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage); } } @@ -85,7 +85,7 @@ class Webhooks extends Action * @param Mail $queueForMails * @return void */ - private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage): void + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage): void { if ($webhook->getAttribute('enabled') !== true) { return; @@ -168,7 +168,7 @@ class Webhooks extends Action $dbForPlatform->purgeCachedDocument('projects', $project->getId()); $this->errors[] = $logs; - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_WEBHOOKS_FAILED, 1) ->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_FAILED), 1) ; @@ -178,13 +178,13 @@ class Webhooks extends Action $webhook->setAttribute('attempts', 0); // Reset attempts on success $dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - $queueForUsage + $queueForStatsUsage ->addMetric(METRIC_WEBHOOKS_SENT, 1) ->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_SENT), 1) ; } - $queueForUsage + $queueForStatsUsage ->setProject($project) ->trigger(); } diff --git a/tests/resources/docker/docker-compose.yml b/tests/resources/docker/docker-compose.yml index 94d506056c..e549ac27a5 100644 --- a/tests/resources/docker/docker-compose.yml +++ b/tests/resources/docker/docker-compose.yml @@ -89,9 +89,9 @@ services: - _APP_FUNCTIONS_MEMORY_SWAP - _APP_EXECUTOR_HOST - appwrite-worker-usage: - entrypoint: worker-usage - container_name: appwrite-worker-usage + appwrite-worker-stats-usage: + entrypoint: worker-stats-usage + container_name: appwrite-worker-stats-usage build: context: . restart: unless-stopped From 6fedefdedac11b289390416785a3e5e3debe0e9d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 05:03:41 +0000 Subject: [PATCH 25/57] fix --- src/Appwrite/Platform/Workers/StatsUsage.php | 40 ++------------------ 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 7ee4cb9890..7ffe9212b3 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\StatsUsageDump; +use Appwrite\Event\UsageDump; use Exception; use Utopia\CLI\Console; use Utopia\Database\DateTime; @@ -21,7 +21,7 @@ class StatsUsage extends Action public static function getName(): string { - return 'stats-usage'; + return 'usage'; } /** @@ -45,7 +45,7 @@ class StatsUsage extends Action /** * @param Message $message * @param callable $getProjectDB - * @param StatsUsageDump $queueForStatsUsageDump + * @param StatsUsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception @@ -184,12 +184,8 @@ class StatsUsage extends Action $deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); $deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); $builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); - $buildsSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS))); - $buildsFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED))); $buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); $buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); - $buildsComputeSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS))); - $buildsComputeFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED))); $executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); $executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); @@ -214,20 +210,6 @@ class StatsUsage extends Action ]; } - if (!empty($buildsSuccess['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_SUCCESS, - 'value' => ($buildsSuccess['value'] * -1), - ]; - } - - if (!empty($buildsFailed['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_FAILED, - 'value' => ($buildsFailed['value'] * -1), - ]; - } - if (!empty($buildsStorage['value'])) { $metrics[] = [ 'key' => METRIC_BUILDS_STORAGE, @@ -242,20 +224,6 @@ class StatsUsage extends Action ]; } - if (!empty($buildsComputeSuccess['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE_SUCCESS, - 'value' => ($buildsComputeSuccess['value'] * -1), - ]; - } - - if (!empty($buildsComputeFailed['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE_FAILED, - 'value' => ($buildsComputeFailed['value'] * -1), - ]; - } - if (!empty($executions['value'])) { $metrics[] = [ 'key' => METRIC_EXECUTIONS, @@ -277,4 +245,4 @@ class StatsUsage extends Action console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } -} +} \ No newline at end of file From d0724812d73bd2288edfc33a5e6a092f45b7bdad Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 05:05:05 +0000 Subject: [PATCH 26/57] fix --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 7ffe9212b3..f2df08db27 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\UsageDump; +use Appwrite\Event\StatsUsageDump; use Exception; use Utopia\CLI\Console; use Utopia\Database\DateTime; From f94d1a5d8240db88974727216f15e00ec904e956 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 05:24:30 +0000 Subject: [PATCH 27/57] update compose.phtml --- app/views/install/compose.phtml | 60 +++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index c69ab60015..eb2b008057 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -649,6 +649,66 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES + appwrite-task-stats-resources: + image: /: + container_name: appwrite-task-stats-resources + entrypoint: stats-resources + <<: *x-logging + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_DATABASE_SHARED_TABLES + - _APP_STATS_RESOURCES_INTERVAL + + appwrite-worker-stats-resources: + image: /: + entrypoint: worker-stats-resources + container_name: appwrite-worker-stats-resources + <<: *x-logging + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_STATS_RESOURCES_INTERVAL + appwrite-worker-stats-usage: image: /: entrypoint: worker-stats-usage From da7a23403861c35f10434360ea1d284e2ba0ab84 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 05:28:45 +0000 Subject: [PATCH 28/57] update --- app/views/install/compose.phtml | 1 - docker-compose.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index eb2b008057..d8f3f649e0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -677,7 +677,6 @@ $image = $this->getParam('image', ''); - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL diff --git a/docker-compose.yml b/docker-compose.yml index 7b05a48cc6..a8193bdec8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -750,7 +750,6 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL From 074f4417ffa27f978841c112c04aa8a32e486c2b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 06:24:10 +0000 Subject: [PATCH 29/57] image transformations metrics and batch writes --- app/init.php | 4 +- .../Platform/Workers/StatsResources.php | 137 ++++++++++++++---- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 3 files changed, 110 insertions(+), 33 deletions(-) diff --git a/app/init.php b/app/init.php index 228625b25b..bdba3175e9 100644 --- a/app/init.php +++ b/app/init.php @@ -237,8 +237,6 @@ const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; const METRIC_WEBHOOK_ID_FAILED = '{webhookInternalId}.webhooks.events.failed'; - - const METRIC_AUTH_METHOD_PHONE = 'auth.method.phone'; const METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE = METRIC_AUTH_METHOD_PHONE . '.{countryCode}'; const METRIC_MESSAGES = 'messages'; @@ -269,6 +267,8 @@ const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; const METRIC_FILES_TRANSFORMATIONS = 'files.transformations'; const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations'; +const METRIC_IMAGES_TRANSFORMATIONS = 'images.transformations'; +const METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS = '{bucketInternalId}.images.transformations'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; const METRIC_FUNCTIONS = 'functions'; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 4f6855a6a1..424ed306b9 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -22,11 +22,20 @@ class StatsResources extends Action 'inf' => '0000-00-00 00:00' ]; + /** + * @var array $documents + * + * Array of documents to batch write + * + */ + protected array $documents = []; + public static function getName(): string { return 'stats-resources'; } + /** * @throws Exception */ @@ -65,6 +74,9 @@ class StatsResources extends Action return; } + // Reset documents for each job + $this->documents = []; + $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); } @@ -120,7 +132,7 @@ class StatsResources extends Action ]; foreach ($metrics as $metric => $value) { - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $value); + $this->createStatsDocuments($region, $metric, $value); } try { @@ -144,6 +156,8 @@ class StatsResources extends Action call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); } + $this->writeDocuments($dbForLogs, $project); + Console::info('End of count for: ' . $project->getId()); } @@ -155,18 +169,65 @@ class StatsResources extends Action $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $files); + $this->createStatsDocuments($region, $metric, $files); $storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual'); $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $storage); + $this->createStatsDocuments($region, $metric, $storage); $totalStorage += $storage; $totalFiles += $files; }); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES, $totalFiles); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES_STORAGE, $totalStorage); + $this->createStatsDocuments($region, METRIC_FILES, $totalFiles); + $this->createStatsDocuments($region, METRIC_FILES_STORAGE, $totalStorage); + } + + /** + * Need separate function to count per period data + */ + protected function countImageTransformations(Database $dbForProject, Database $dbForLogs, string $region) + { + $totalImageTransformations = 0; + $totalDailyImageTransformations = 0; + $totalHourlyImageTransformations = 0; + $this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalDailyImageTransformations, &$totalHourlyImageTransformations, &$totalImageTransformations) { + $imageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [ + Query::isNotNull('transformedAt') + ]); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS); + $this->createStatsDocuments($region, $metric, $imageTransformations, 'inf'); + $totalImageTransformations += $imageTransformations; + + // hourly + $time = \date($this->periods['1h'], \time()); + $start = $time; + $end = (new \DateTime($start))->format('Y-m-d H:59:59'); + $hourlyImageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [ + Query::greaterThanEqual('transformedAt', $start), + Query::lessThanEqual('transformedAt', $end), + ]); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS); + $this->createStatsDocuments($region, $metric, $hourlyImageTransformations, '1h'); + $totalHourlyImageTransformations += $hourlyImageTransformations; + + // daily + $time = \date($this->periods['1d'], \time()); + $start = $time; + $end = (new \DateTime($start))->format('Y-m-d 11:59:59'); + + $dailyImageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [ + Query::greaterThanEqual('transformedAt', $start), + Query::lessThanEqual('transformedAt', $end), + ]); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS); + $this->createStatsDocuments($region, $metric, $dailyImageTransformations, '1d'); + $totalDailyImageTransformations += $dailyImageTransformations; + + }); + + $this->createStatsDocuments($region, METRIC_IMAGES_TRANSFORMATIONS, $totalImageTransformations, 'inf'); + } protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region) @@ -178,7 +239,7 @@ class StatsResources extends Action $collections = $dbForProject->count('database_' . $database->getInternalId()); $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $collections); + $this->createStatsDocuments($region, $metric, $collections); $documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region); @@ -186,8 +247,8 @@ class StatsResources extends Action $totalCollections += $collections; }); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_COLLECTIONS, $totalCollections); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DOCUMENTS, $totalDocuments); + $this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections); + $this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments); } protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int { @@ -196,13 +257,13 @@ class StatsResources extends Action $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $documents); + $this->createStatsDocuments($region, $metric, $documents); $databaseDocuments += $documents; }); $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS); - $this->createOrUpdateMetric($dbForLogs, $region, $metric, $databaseDocuments); + $this->createStatsDocuments($region, $metric, $databaseDocuments); return $databaseDocuments; } @@ -211,13 +272,13 @@ class StatsResources extends Action { $deploymentsStorage = $dbForProject->sum('deployments', 'size'); $buildsStorage = $dbForProject->sum('builds', 'size'); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS_STORAGE, $buildsStorage); + $this->createStatsDocuments($region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); + $this->createStatsDocuments($region, METRIC_BUILDS_STORAGE, $buildsStorage); $deployments = $dbForProject->count('deployments'); $builds = $dbForProject->count('builds'); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS, $deployments); - $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS, $builds); + $this->createStatsDocuments($region, METRIC_DEPLOYMENTS, $deployments); + $this->createStatsDocuments($region, METRIC_BUILDS, $builds); $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { @@ -225,19 +286,19 @@ class StatsResources extends Action Query::equal('resourceInternalId', [$function->getInternalId()]), Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), ]); - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage); + $this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage); $functionDeployments = $dbForProject->count('deployments', [ Query::equal('resourceInternalId', [$function->getInternalId()]), Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), ]); - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments); + $this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments); /** * As deployments and builds have 1-1 relationship, * the count for one should match the other */ - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments); + $this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments); $functionBuildsStorage = 0; @@ -249,29 +310,45 @@ class StatsResources extends Action $functionBuildsStorage += $build->getAttribute('size', 0); }); - $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); + $this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); }); } - protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value) + protected function createStatsDocuments(string $region, string $metric, int $value, ?string $period = null) { - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : \date($format, \time()); - $id = \md5("{$time}_{$period}_{$metric}"); - $current = $dbForLogs->getDocument('usage', $id); - if ($current->isEmpty()) { - $dbForLogs->createDocument('usage', new Document([ + if ($period === null) { + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : \date($format, \time()); + $id = \md5("{$time}_{$period}_{$metric}"); + + $this->documents[] = new Document([ '$id' => $id, 'metric' => $metric, 'period' => $period, 'region' => $region, 'value' => $value, - ])); - Console::success('Usage logs created for metric: ' . $metric . ' period:'. $period); - } else { - $dbForLogs->updateDocument('usage', $id, $current->setAttribute('value', $value)); - Console::success('Usage logs updated for metric: ' . $metric . ' period:'. $period); + ]); } + } else { + $time = 'inf' === $period ? null : \date($this->periods[$period], \time()); + $id = \md5("{$time}_{$period}_{$metric}"); + $this->documents[] = new Document([ + '$id' => $id, + 'metric' => $metric, + 'period' => $period, + 'region' => $region, + 'value' => $value, + ]); } } + + protected function writeDocuments(Database $dbForLogs, Document $project): void + { + $dbForLogs->createOrUpdateDocuments( + 'stats', + $this->documents + ); + $this->documents = []; + Console::success('Stats written to logs db for project: ' . $project->getId() . '(' . $project->getInternalId() . ')'); + } } diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index f2df08db27..1bdf44d607 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -245,4 +245,4 @@ class StatsUsage extends Action console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } -} \ No newline at end of file +} From 6eb4d3f021754199003d24eb40d8e5d9a36f9c2a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 06:40:49 +0000 Subject: [PATCH 30/57] reset dockerfile --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 01ad5d3fba..2bb9f80d9e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -88,7 +88,9 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-stats-usage && \ chmod +x /usr/local/bin/worker-stats-usage-dump && \ chmod +x /usr/local/bin/stats-resources && \ - chmod +x /usr/local/bin/worker-stats-resources + chmod +x /usr/local/bin/worker-stats-resources && \ + chmod +x /usr/local/bin/worker-usage && \ + chmod +x /usr/local/bin/worker-usage-dump # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ From 95845f8c0ea6a147dd0796451925be861fe0c0f4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 06:42:57 +0000 Subject: [PATCH 31/57] update env name --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 58cd9c13fb..2d5a8dbbab 100644 --- a/.env +++ b/.env @@ -86,7 +86,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_USAGE_AGGREGATION_INTERVAL=30 -_APP_USAGE_COUNT_INTERVAL=3600 +_APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled From d31a358392c83daa6b0ee020ddaa67ada1df2636 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 07:07:10 +0000 Subject: [PATCH 32/57] Temp: everything to be removed --- app/controllers/api/health.php | 4 +- app/worker.php | 15 + bin/worker-usage | 3 + bin/worker-usage-dump | 3 + docker-compose.yml | 64 +++++ src/Appwrite/Event/Event.php | 8 + src/Appwrite/Event/StatsUsage.php | 4 +- src/Appwrite/Event/Usage.php | 78 +++++ src/Appwrite/Event/UsageDump.php | 44 +++ src/Appwrite/Platform/Services/Workers.php | 8 + src/Appwrite/Platform/Workers/Usage.php | 288 +++++++++++++++++++ src/Appwrite/Platform/Workers/UsageDump.php | 301 ++++++++++++++++++++ 12 files changed, 816 insertions(+), 4 deletions(-) create mode 100644 bin/worker-usage create mode 100644 bin/worker-usage-dump create mode 100644 src/Appwrite/Event/Usage.php create mode 100644 src/Appwrite/Event/UsageDump.php create mode 100644 src/Appwrite/Platform/Workers/Usage.php create mode 100644 src/Appwrite/Platform/Workers/UsageDump.php diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index ab3551fcd9..a4f6586262 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -788,7 +788,7 @@ App::get('/v1/health/queue/usage') ->action(function (int|string $threshold, Connection $queue, Response $response) { $threshold = \intval($threshold); - $client = new Client(Event::USAGE_QUEUE_NAME, $queue); + $client = new Client(Event::STATS_USAGE_QUEUE_NAME, $queue); $size = $client->getQueueSize(); if ($size >= $threshold) { @@ -995,7 +995,7 @@ App::get('/v1/health/queue/failed/:name') Event::AUDITS_QUEUE_NAME, Event::MAILS_QUEUE_NAME, Event::FUNCTIONS_QUEUE_NAME, - Event::USAGE_QUEUE_NAME, + Event::STATS_USAGE_QUEUE_NAME, Event::STATS_USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_QUEUE_NAME, Event::CERTIFICATES_QUEUE_NAME, diff --git a/app/worker.php b/app/worker.php index 415d6a6f27..d57a2712a0 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,8 +13,12 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +/** remove */ use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; +/** /remove */ +use Appwrite\Event\Usage; +use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; @@ -257,6 +261,17 @@ Server::setResource('timelimit', function (\Redis $redis) { Server::setResource('log', fn () => new Log()); +/** remove */ +Server::setResource('queueForUsage', function (Connection $queue) { + return new Usage($queue); + return new StatsUsage($queue); +}, ['queue']); + +Server::setResource('queueForUsageDump', function (Connection $queue) { + return new UsageDump($queue); + return new StatsUsageDump($queue); +}, ['queue']); +/** /remove */ Server::setResource('queueForStatsUsage', function (Connection $queue) { return new StatsUsage($queue); }, ['queue']); diff --git a/bin/worker-usage b/bin/worker-usage new file mode 100644 index 0000000000..e39ce8477c --- /dev/null +++ b/bin/worker-usage @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php usage $@ \ No newline at end of file diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump new file mode 100644 index 0000000000..43ca87fcb3 --- /dev/null +++ b/bin/worker-usage-dump @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a8193bdec8..4ea1408d9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -845,7 +845,71 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES + + # remove + appwrite-worker-usage: + entrypoint: worker-usage + <<: *x-logging + container_name: appwrite-worker-usage + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_DATABASE_SHARED_TABLES + appwrite-worker-usage-dump: + entrypoint: worker-usage-dump + <<: *x-logging + container_name: appwrite-worker-usage-dump + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_DATABASE_SHARED_TABLES + # /remove + appwrite-task-scheduler-functions: entrypoint: schedule-functions <<: *x-logging diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index a1c5abd1d8..f6638d2cf6 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -24,12 +24,20 @@ class Event public const FUNCTIONS_QUEUE_NAME = 'v1-functions'; public const FUNCTIONS_CLASS_NAME = 'FunctionsV1'; + /** remove */ public const USAGE_QUEUE_NAME = 'v1-usage'; public const USAGE_CLASS_NAME = 'UsageV1'; + public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; + public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; + /** /remove */ + public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources'; public const STATS_RESOURCES_CLASS_NAME = 'StatsResources'; + public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage'; + public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1'; + public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump'; public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1'; diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php index 7f54370fca..a1c027f797 100644 --- a/src/Appwrite/Event/StatsUsage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -16,8 +16,8 @@ class StatsUsage extends Event parent::__construct($connection); $this - ->setQueue(Event::USAGE_QUEUE_NAME) - ->setClass(Event::USAGE_CLASS_NAME); + ->setQueue(Event::STATS_USAGE_QUEUE_NAME) + ->setClass(Event::STATS_USAGE_CLASS_NAME); } /** diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php new file mode 100644 index 0000000000..5609859f37 --- /dev/null +++ b/src/Appwrite/Event/Usage.php @@ -0,0 +1,78 @@ +setQueue(Event::USAGE_QUEUE_NAME) + ->setClass(Event::USAGE_CLASS_NAME); + } + + /** + * Add reduce. + * + * @param Document $document + * @return self + */ + public function addReduce(Document $document): self + { + $this->reduce[] = $document; + + return $this; + } + + /** + * Add metric. + * + * @param string $key + * @param int $value + * @return self + */ + public function addMetric(string $key, int $value): self + { + + $this->metrics[] = [ + 'key' => $key, + 'value' => $value, + ]; + + return $this; + } + + /** + * Prepare the payload for the usage event. + * + * @return array + */ + protected function preparePayload(): array + { + return [ + 'project' => $this->project, + 'reduce' => $this->reduce, + 'metrics' => $this->metrics, + ]; + } + + /** + * Sends metrics to the usage worker. + * + * @return string|bool + */ + public function trigger(): string|bool + { + parent::trigger(); + $this->metrics = []; + return true; + } +} diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php new file mode 100644 index 0000000000..6f44de4eda --- /dev/null +++ b/src/Appwrite/Event/UsageDump.php @@ -0,0 +1,44 @@ +setQueue(Event::USAGE_DUMP_QUEUE_NAME) + ->setClass(Event::USAGE_DUMP_CLASS_NAME); + } + + /** + * Add Stats. + * + * @param array $stats + * @return self + */ + public function setStats(array $stats): self + { + $this->stats = $stats; + + return $this; + } + + /** + * Prepare the payload for the usage dump event. + * + * @return array + */ + protected function preparePayload(): array + { + return [ + 'stats' => $this->stats, + ]; + } +} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 4f4095aca4..e121ee35f7 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -14,6 +14,10 @@ use Appwrite\Platform\Workers\Migrations; use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\StatsUsage; use Appwrite\Platform\Workers\StatsUsageDump; +/** remove */ +use Appwrite\Platform\Workers\Usage; +use Appwrite\Platform\Workers\UsageDump; +/** /remove */ use Appwrite\Platform\Workers\Webhooks; use Utopia\Platform\Service; @@ -36,6 +40,10 @@ class Workers extends Service ->addAction(StatsUsage::getName(), new StatsUsage()) ->addAction(Migrations::getName(), new Migrations()) ->addAction(StatsResources::getName(), new StatsResources()) + /** Remove */ + ->addAction(UsageDump::getName(), new UsageDump()) + ->addAction(Usage::getName(), new Usage()) + /** /remove */ ; } } diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php new file mode 100644 index 0000000000..3f7428d0dd --- /dev/null +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -0,0 +1,288 @@ +desc('Usage worker') + ->inject('message') + ->inject('project') + ->inject('getProjectDB') + ->inject('queueForUsageDump') + ->callback([$this, 'action']); + + $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); + $this->lastTriggeredTime = time(); + } + + /** + * @param Message $message + * @param Document $project + * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump + * @return void + * @throws \Utopia\Database\Exception + * @throws Exception + */ + public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump): void + { + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + if (empty($project->getAttribute('database'))) { + var_dump($payload); + return; + } + + $projectId = $project->getInternalId(); + foreach ($payload['reduce'] ?? [] as $document) { + if (empty($document)) { + continue; + } + + $this->reduce( + project: $project, + document: new Document($document), + metrics: $payload['metrics'], + getProjectDB: $getProjectDB + ); + } + + + $this->stats[$projectId]['project'] = [ + '$id' => $project->getId(), + '$internalId' => $project->getInternalId(), + 'database' => $project->getAttribute('database'), + ]; + $this->stats[$projectId]['receivedAt'] = DateTime::now(); + foreach ($payload['metrics'] ?? [] as $metric) { + $this->keys++; + if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { + $this->stats[$projectId]['keys'][$metric['key']] = $metric['value']; + continue; + } + + $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; + } + + // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) + if ( + $this->keys >= self::KEYS_THRESHOLD || + (time() - $this->lastTriggeredTime > $this->aggregationInterval && $this->keys > 0) + ) { + Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); + + $queueForUsageDump + ->setStats($this->stats) + ->trigger(); + + $this->stats = []; + $this->keys = 0; + $this->lastTriggeredTime = time(); + } + } + + /** + * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files. + * When we remove a parent document we need to deduct his children aggregation from the project scope. + * @param Document $project + * @param Document $document + * @param array $metrics + * @param callable $getProjectDB + * @return void + */ + private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void + { + $dbForProject = $getProjectDB($project); + + try { + switch (true) { + case $document->getCollection() === 'users': // users + $sessions = count($document->getAttribute(METRIC_SESSIONS, 0)); + if (!empty($sessions)) { + $metrics[] = [ + 'key' => METRIC_SESSIONS, + 'value' => ($sessions * -1), + ]; + } + break; + case $document->getCollection() === 'databases': // databases + $collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS))); + $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS))); + if (!empty($collections['value'])) { + $metrics[] = [ + 'key' => METRIC_COLLECTIONS, + 'value' => ($collections['value'] * -1), + ]; + } + + if (!empty($documents['value'])) { + $metrics[] = [ + 'key' => METRIC_DOCUMENTS, + 'value' => ($documents['value'] * -1), + ]; + } + break; + case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS))); + + if (!empty($documents['value'])) { + $metrics[] = [ + 'key' => METRIC_DOCUMENTS, + 'value' => ($documents['value'] * -1), + ]; + $metrics[] = [ + 'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), + 'value' => ($documents['value'] * -1), + ]; + } + break; + + case $document->getCollection() === 'buckets': + $files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES))); + $storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE))); + + if (!empty($files['value'])) { + $metrics[] = [ + 'key' => METRIC_FILES, + 'value' => ($files['value'] * -1), + ]; + } + + if (!empty($storage['value'])) { + $metrics[] = [ + 'key' => METRIC_FILES_STORAGE, + 'value' => ($storage['value'] * -1), + ]; + } + break; + + case $document->getCollection() === 'functions': + $deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); + $deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); + $builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); + $buildsSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS))); + $buildsFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED))); + $buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); + $buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); + $buildsComputeSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS))); + $buildsComputeFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED))); + $executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); + $executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); + + if (!empty($deployments['value'])) { + $metrics[] = [ + 'key' => METRIC_DEPLOYMENTS, + 'value' => ($deployments['value'] * -1), + ]; + } + + if (!empty($deploymentsStorage['value'])) { + $metrics[] = [ + 'key' => METRIC_DEPLOYMENTS_STORAGE, + 'value' => ($deploymentsStorage['value'] * -1), + ]; + } + + if (!empty($builds['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS, + 'value' => ($builds['value'] * -1), + ]; + } + + if (!empty($buildsSuccess['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_SUCCESS, + 'value' => ($buildsSuccess['value'] * -1), + ]; + } + + if (!empty($buildsFailed['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_FAILED, + 'value' => ($buildsFailed['value'] * -1), + ]; + } + + if (!empty($buildsStorage['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_STORAGE, + 'value' => ($buildsStorage['value'] * -1), + ]; + } + + if (!empty($buildsCompute['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_COMPUTE, + 'value' => ($buildsCompute['value'] * -1), + ]; + } + + if (!empty($buildsComputeSuccess['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_COMPUTE_SUCCESS, + 'value' => ($buildsComputeSuccess['value'] * -1), + ]; + } + + if (!empty($buildsComputeFailed['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_COMPUTE_FAILED, + 'value' => ($buildsComputeFailed['value'] * -1), + ]; + } + + if (!empty($executions['value'])) { + $metrics[] = [ + 'key' => METRIC_EXECUTIONS, + 'value' => ($executions['value'] * -1), + ]; + } + + if (!empty($executionsCompute['value'])) { + $metrics[] = [ + 'key' => METRIC_EXECUTIONS_COMPUTE, + 'value' => ($executionsCompute['value'] * -1), + ]; + } + break; + default: + break; + } + } catch (\Throwable $e) { + console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); + } + } +} diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php new file mode 100644 index 0000000000..bb1d605442 --- /dev/null +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -0,0 +1,301 @@ + 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'usage-dump'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + $this + ->inject('message') + ->inject('getProjectDB') + ->callback([$this, 'action']); + } + + /** + * @param Message $message + * @param callable $getProjectDB + * @return void + * @throws Exception + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, callable $getProjectDB): void + { + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + try { + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + $numberOfKeys = !empty($stats['keys']) ? \count($stats['keys']) : 0; + $receivedAt = $stats['receivedAt'] ?? 'NONE'; + if ($numberOfKeys === 0) { + continue; + } + + $dbForProject = $getProjectDB($project); + $projectDocuments = []; + $databaseCache = []; + $collectionSizeCache = []; + + Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys . ' Started'); + $start = \microtime(true); + + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : \date($format, \time()); + $id = \md5("{$time}_{$period}_{$key}"); + + if (\str_contains($key, METRIC_DATABASES_STORAGE)) { + $this->handleDatabaseStorage( + $id, + $key, + $time, + $period, + $dbForProject, + $projectDocuments, + $databaseCache, + $collectionSizeCache + ); + continue; + } + + $projectDocuments[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + } + } + + $dbForProject->createOrUpdateDocumentsWithIncrease( + collection: 'stats', + attribute: 'value', + documents: $projectDocuments + ); + + $end = \microtime(true); + Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s'); + } + } catch (\Exception $e) { + Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage()); + } + } + + private function handleDatabaseStorage( + string $id, + string $key, + ?string $time, + string $period, + Database $dbForProject, + array &$projectDocuments, + array &$databaseCache, + array &$collectionSizeCache, + ): void { + $data = \explode('.', $key); + $value = 0; + $previousValue = 0; + + try { + $previousValue = $dbForProject + ->getDocument('stats', $id) + ->getAttribute('value', 0); + } catch (\Exception) { + // No previous value + } + + switch (\count($data)) { + case METRIC_COLLECTION_LEVEL_STORAGE: + $databaseInternalId = $data[0]; + $collectionInternalId = $data[1]; + $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}"; + + if (!isset($collectionSizeCache[$collectionId])) { + try { + $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $collectionSizeCache[$collectionId] = 0; + } + } + + $value = $collectionSizeCache[$collectionId]; + + $diff = $value - $previousValue; + if ($diff === 0) { + break; + } + + $keys = [ + $key, + \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), + METRIC_DATABASES_STORAGE + ]; + + foreach ($keys as $metric) { + $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); + } + break; + + case METRIC_DATABASE_LEVEL_STORAGE: + $databaseInternalId = $data[0]; + $databaseId = "database_{$databaseInternalId}"; + + if (!isset($databaseCache[$databaseId])) { + try { + $databaseCache[$databaseId] = $dbForProject->find($databaseId); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $databaseCache[$databaseId] = []; + } + } + + foreach ($databaseCache[$databaseId] as $collection) { + $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; + + if (!isset($collectionSizeCache[$collectionId])) { + try { + $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $collectionSizeCache[$collectionId] = 0; + } + } + $value += $collectionSizeCache[$collectionId]; + } + + $diff = $value - $previousValue; + if ($diff === 0) { + break; + } + + $keys = [ + \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), + METRIC_DATABASES_STORAGE + ]; + + foreach ($keys as $metric) { + $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); + } + break; + + case METRIC_PROJECT_LEVEL_STORAGE: + if (!isset($databaseCache['*'])) { + try { + $databaseCache['*'] = $dbForProject->find('databases'); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $databaseCache['*'] = []; + } + } + + foreach ($databaseCache['*'] as $database) { + $databaseId = "database_{$database->getInternalId()}"; + if (!isset($databaseCache[$databaseId])) { + try { + $databaseCache[$databaseId] = $dbForProject->find($databaseId); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $databaseCache[$databaseId] = []; + } + } + + foreach ($databaseCache[$databaseId] as $collection) { + $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; + + if (!isset($collectionSizeCache[$collectionId])) { + try { + $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $collectionSizeCache[$collectionId] = 0; + } + } + $value += $collectionSizeCache[$collectionId]; + } + } + + $diff = $value - $previousValue; + if ($diff === 0) { + break; + } + + $keys = [ + METRIC_DATABASES_STORAGE + ]; + + foreach ($keys as $metric) { + $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); + } + + break; + } + } + + private function createStatsDocument( + string $id, + string $period, + ?string $time, + string $key, + int $diff, + ): Document { + return new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + } +} From 71c265d4e8feaa902c9070062ecf0deccf2711bb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 07:12:15 +0000 Subject: [PATCH 33/57] fix wrong comment --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index d57a2712a0..7168d4e439 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,12 +13,12 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; -/** remove */ use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; -/** /remove */ +/** remove */ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; +/** /remove */ use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; From 978f22b56d5a62dc203ae315a05ced8f3f7e5d19 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 07:15:24 +0000 Subject: [PATCH 34/57] rename transformed metric --- app/init.php | 4 ++-- src/Appwrite/Platform/Workers/StatsResources.php | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/init.php b/app/init.php index bdba3175e9..930bae66c3 100644 --- a/app/init.php +++ b/app/init.php @@ -267,8 +267,8 @@ const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; const METRIC_FILES_TRANSFORMATIONS = 'files.transformations'; const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations'; -const METRIC_IMAGES_TRANSFORMATIONS = 'images.transformations'; -const METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS = '{bucketInternalId}.images.transformations'; +const METRIC_FILES_IMAGES_TRANSFORMED = 'files.imagesTransformed'; +const METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED = '{bucketInternalId}.files.imagesTransformed'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; const METRIC_FUNCTIONS = 'functions'; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 424ed306b9..62992bc7b3 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -195,7 +195,7 @@ class StatsResources extends Action $imageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [ Query::isNotNull('transformedAt') ]); - $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED); $this->createStatsDocuments($region, $metric, $imageTransformations, 'inf'); $totalImageTransformations += $imageTransformations; @@ -207,7 +207,7 @@ class StatsResources extends Action Query::greaterThanEqual('transformedAt', $start), Query::lessThanEqual('transformedAt', $end), ]); - $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED); $this->createStatsDocuments($region, $metric, $hourlyImageTransformations, '1h'); $totalHourlyImageTransformations += $hourlyImageTransformations; @@ -220,13 +220,13 @@ class StatsResources extends Action Query::greaterThanEqual('transformedAt', $start), Query::lessThanEqual('transformedAt', $end), ]); - $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_IMAGES_TRANSFORMATIONS); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED); $this->createStatsDocuments($region, $metric, $dailyImageTransformations, '1d'); $totalDailyImageTransformations += $dailyImageTransformations; }); - $this->createStatsDocuments($region, METRIC_IMAGES_TRANSFORMATIONS, $totalImageTransformations, 'inf'); + $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations, 'inf'); } From 5b91b5cec17049fb7f6a7a485d27ad0e494490a0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:10:45 +0000 Subject: [PATCH 35/57] env to configure dual write --- .env | 1 + docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.env b/.env index 2d5a8dbbab..0b2f129f08 100644 --- a/.env +++ b/.env @@ -111,3 +111,4 @@ _APP_MESSAGE_PUSH_TEST_DSN= _APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 _APP_PROJECT_REGIONS=default _APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000 +_APP_STATS_USAGE_DUAL_WRITING_DBS= \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4ea1408d9c..64ae108f7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -845,6 +845,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES + - _APP_STATS_USAGE_DUAL_WRITING_DBS # remove appwrite-worker-usage: From 5052fd8528076b3aabeb7709bdcbc8f016b8693f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:11:03 +0000 Subject: [PATCH 36/57] get latest usageDump and dual write --- .../Platform/Workers/StatsUsageDump.php | 380 ++++++++---------- 1 file changed, 175 insertions(+), 205 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index fa31691a18..d373c65a88 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -7,7 +7,6 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception\NotFound; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; @@ -64,6 +63,11 @@ class StatsUsageDump extends Action '.builds.storage', ]; + /** + * @var callable + */ + protected mixed $getLogsDB; + protected array $periods = [ '1h' => 'Y-m-d H:00', '1d' => 'Y-m-d 00:00', @@ -90,59 +94,56 @@ class StatsUsageDump extends Action /** * @param Message $message * @param callable $getProjectDB + * @param callable $getLogsDB * @return void * @throws Exception * @throws \Utopia\Database\Exception */ public function action(Message $message, callable $getProjectDB, callable $getLogsDB): void { + $this->getLogsDB = $getLogsDB; $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } - try { - foreach ($payload['stats'] ?? [] as $stats) { - $project = new Document($stats['project'] ?? []); - $numberOfKeys = !empty($stats['keys']) ? \count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? 'NONE'; - if ($numberOfKeys === 0) { - continue; - } + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + + /** + * End temp bug fallback + */ + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + $receivedAt = $stats['receivedAt'] ?? 'NONE'; + if ($numberOfKeys === 0) { + continue; + } + + console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + + try { + /** @var \Utopia\Database\Database $dbForProject */ $dbForProject = $getProjectDB($project); - $dbForLogs = $getLogsDB($project); - $projectDocuments = []; - $databaseCache = []; - $collectionSizeCache = []; - - Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys . ' Started'); - $start = \microtime(true); - foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; } + if (str_contains($key, METRIC_DATABASES_STORAGE)) { + try { + $this->handleDatabaseStorage($key, $dbForProject, $project); + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); + } + continue; + } + foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : \date($format, \time()); + $time = 'inf' === $period ? null : date($format, time()); $id = \md5("{$time}_{$period}_{$key}"); - if (\str_contains($key, METRIC_DATABASES_STORAGE)) { - $this->handleDatabaseStorage( - $id, - $key, - $time, - $period, - $dbForProject, - $projectDocuments, - $databaseCache, - $collectionSizeCache - ); - continue; - } - - $projectDocuments[] = new Document([ + $document = new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -150,223 +151,192 @@ class StatsUsageDump extends Action 'value' => $value, 'region' => System::getEnv('_APP_REGION', 'default'), ]); + $dbForProject->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + [$document] + ); + + $this->writeToLogsDB($project, $document); } } - - /** - * Create clone to dual write - * This is required as first request to db - * modifies the document's details like internal ID - * which will conflict in new DB - */ - $clonedProjectDoucments = []; - foreach ($projectDocuments as $document) { - if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) { - continue; - } - foreach ($this->skipParentIdMetrics as $skipMetric) { - if (str_ends_with($document->getAttribute('metric'), $skipMetric)) { - continue; - } - } - $clonedProjectDoucments[] = new Document($document->getArrayCopy()); - } - - $dbForProject->createOrUpdateDocumentsWithIncrease( - collection: 'stats', - attribute: 'value', - documents: $projectDocuments - ); - - $dbForLogs->createOrUpdateDocumentsWithIncrease( - collection: 'usage', - attribute: 'value', - documents: $clonedProjectDoucments - ); - $end = \microtime(true); - Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s'); + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); } - } catch (\Exception $e) { - Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage()); } } - private function handleDatabaseStorage( - string $id, - string $key, - ?string $time, - string $period, - Database $dbForProject, - array &$projectDocuments, - array &$databaseCache, - array &$collectionSizeCache, - ): void { - $data = \explode('.', $key); - $value = 0; - $previousValue = 0; + private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project): void + { + $data = explode('.', $key); + $start = microtime(true); - try { - $previousValue = $dbForProject - ->getDocument('stats', $id) - ->getAttribute('value', 0); - } catch (\Exception) { - // No previous value - } + $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) { + $id = \md5("{$time}_{$period}_{$key}"); - switch (\count($data)) { - case METRIC_COLLECTION_LEVEL_STORAGE: - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; - $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}"; + $document = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + $dbForProject->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + [$document] + ); + $this->writeToLogsDB($project, $document); + }; + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + $value = 0; + $previousValue = 0; + try { + $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); + } catch (\Exception $e) { + // No previous value + } + + switch (count($data)) { + // Collection Level + case METRIC_COLLECTION_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; + $collectionInternalId = $data[1]; - if (!isset($collectionSizeCache[$collectionId])) { try { - $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $collectionSizeCache[$collectionId] = 0; } - } - $value = $collectionSizeCache[$collectionId]; + // Compare with previous value + $diff = $value - $previousValue; - $diff = $value - $previousValue; - if ($diff === 0) { + if ($diff === 0) { + break; + } + + // Update Collection + $updateMetric($dbForProject, $project, $diff, $key, $period, $time); + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); break; - } + // Database Level + case METRIC_DATABASE_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; - $keys = [ - $key, - \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), - METRIC_DATABASES_STORAGE - ]; - - foreach ($keys as $metric) { - $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); - } - break; - - case METRIC_DATABASE_LEVEL_STORAGE: - $databaseInternalId = $data[0]; - $databaseId = "database_{$databaseInternalId}"; - - if (!isset($databaseCache[$databaseId])) { + $collections = []; try { - $databaseCache[$databaseId] = $dbForProject->find($databaseId); + $collections = $dbForProject->find('database_' . $databaseInternalId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Database not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $databaseCache[$databaseId] = []; } - } - foreach ($databaseCache[$databaseId] as $collection) { - $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; - - if (!isset($collectionSizeCache[$collectionId])) { + foreach ($collections as $collection) { try { - $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $collectionSizeCache[$collectionId] = 0; } } - $value += $collectionSizeCache[$collectionId]; - } - $diff = $value - $previousValue; - if ($diff === 0) { + $diff = $value - $previousValue; + + if ($diff === 0) { + break; + } + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); break; - } + // Project Level + case METRIC_PROJECT_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); + // Get all project databases + $databases = $dbForProject->find('database'); - $keys = [ - \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), - METRIC_DATABASES_STORAGE - ]; + // Recalculate all databases + foreach ($databases as $database) { + $collections = $dbForProject->find('database_' . $database->getInternalId()); - foreach ($keys as $metric) { - $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); - } - break; - - case METRIC_PROJECT_LEVEL_STORAGE: - if (!isset($databaseCache['*'])) { - try { - $databaseCache['*'] = $dbForProject->find('databases'); - } catch (\Exception $e) { - if (!$e instanceof NotFound) { - throw $e; - } - $databaseCache['*'] = []; - } - } - - foreach ($databaseCache['*'] as $database) { - $databaseId = "database_{$database->getInternalId()}"; - if (!isset($databaseCache[$databaseId])) { - try { - $databaseCache[$databaseId] = $dbForProject->find($databaseId); - } catch (\Exception $e) { - if (!$e instanceof NotFound) { - throw $e; - } - $databaseCache[$databaseId] = []; - } - } - - foreach ($databaseCache[$databaseId] as $collection) { - $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; - - if (!isset($collectionSizeCache[$collectionId])) { + foreach ($collections as $collection) { try { - $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); + $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { throw $e; } - $collectionSizeCache[$collectionId] = 0; } } - $value += $collectionSizeCache[$collectionId]; } - } - $diff = $value - $previousValue; - if ($diff === 0) { + $diff = $value - $previousValue; + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); break; - } - - $keys = [ - METRIC_DATABASES_STORAGE - ]; - - foreach ($keys as $metric) { - $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); - } - - break; + } } + + $end = microtime(true); + + console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); } - private function createStatsDocument( - string $id, - string $period, - ?string $time, - string $key, - int $diff, - ): Document { - return new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); + protected function writeToLogsDB(Document $project, Document $document) + { + $databasesToDualWrite = explode(',', System::getEnv('_APP_STATS_USAGE_DUAL_WRITING_DBS', '')); + + $db = $project->getAttribute('database'); + if (!in_array($db, $databasesToDualWrite)) { + return; + } + + /** @var \Utopia\Database\Database $dbForLogs*/ + $dbForLogs = call_user_func($this->getLogsDB, $project); + + if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) { + return; + } + foreach ($this->skipParentIdMetrics as $skipMetric) { + if (str_ends_with($document->getAttribute('metric'), $skipMetric)) { + return; + } + } + + $dbForLogs->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + [$document] + ); } } From 292e8591335be83ce4be4defd64e8e8d3cb8b628 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:17:27 +0000 Subject: [PATCH 37/57] rename metric --- app/init.php | 2 +- src/Appwrite/Platform/Workers/StatsResources.php | 2 +- src/Appwrite/Platform/Workers/StatsUsageDump.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init.php b/app/init.php index 8c1c3c47a6..910a7f0f22 100644 --- a/app/init.php +++ b/app/init.php @@ -302,7 +302,7 @@ const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.execution const METRIC_NETWORK_REQUESTS = 'network.requests'; const METRIC_NETWORK_INBOUND = 'network.inbound'; const METRIC_NETWORK_OUTBOUND = 'network.outbound'; -const METRIC_USERS_ACTIVE = 'users.mau'; +const METRIC_MAU = 'users.mau'; const METRIC_WEBHOOKS = 'webhooks'; const METRIC_PLATFORMS = 'platforms'; const METRIC_PROVIDERS = 'providers'; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 62992bc7b3..fc95cfdd58 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -123,7 +123,7 @@ class StatsResources extends Action METRIC_FUNCTIONS => $functions, METRIC_TEAMS => $teams, METRIC_MESSAGES => $messages, - METRIC_USERS_ACTIVE => $usersActive, + METRIC_MAU => $usersActive, METRIC_WEBHOOKS => $webhooks, METRIC_PLATFORMS => $platforms, METRIC_PROVIDERS => $providers, diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index d373c65a88..6d3e2dd15e 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -32,7 +32,7 @@ class StatsUsageDump extends Action METRIC_FUNCTIONS => true, METRIC_TEAMS => true, METRIC_MESSAGES => true, - METRIC_USERS_ACTIVE => true, + METRIC_MAU => true, METRIC_WEBHOOKS => true, METRIC_PLATFORMS => true, METRIC_PROVIDERS => true, From 6b930129e5afd581cb14ccf44e4ccde5fa13528b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:26:00 +0000 Subject: [PATCH 38/57] add DAU and WAU metric --- app/init.php | 2 ++ src/Appwrite/Platform/Workers/StatsResources.php | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 910a7f0f22..09560e9d4a 100644 --- a/app/init.php +++ b/app/init.php @@ -303,6 +303,8 @@ const METRIC_NETWORK_REQUESTS = 'network.requests'; const METRIC_NETWORK_INBOUND = 'network.inbound'; const METRIC_NETWORK_OUTBOUND = 'network.outbound'; const METRIC_MAU = 'users.mau'; +const METRIC_DAU = 'users.dau'; +const METRIC_WAU = 'users.wau'; const METRIC_WEBHOOKS = 'webhooks'; const METRIC_PLATFORMS = 'platforms'; const METRIC_PROVIDERS = 'providers'; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index fc95cfdd58..668d71d703 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -107,9 +107,17 @@ class StatsResources extends Action $users = $dbForProject->count('users'); $last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00'); - $usersActive = $dbForProject->count('users', [ + $usersMAU = $dbForProject->count('users', [ Query::greaterThanEqual('accessedAt', $last30Days) ]); + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'))->format('Y-m-d h:m:00'); + $usersDAU = $dbForProject->count('users', [ + Query::greaterThanEqual('accessedAt', $last24Hours) + ]); + $last7Days = (new \DateTime())->sub(\DateInterval::createFromDateString('7 days'))->format('Y-m-d 00:00:00'); + $usersWAU = $dbForProject->count('users', [ + Query::greaterThanEqual('accessedAt', $last7Days) + ]); $teams = $dbForProject->count('teams'); $functions = $dbForProject->count('functions'); $messages = $dbForProject->count('messages'); @@ -123,7 +131,9 @@ class StatsResources extends Action METRIC_FUNCTIONS => $functions, METRIC_TEAMS => $teams, METRIC_MESSAGES => $messages, - METRIC_MAU => $usersActive, + METRIC_MAU => $usersMAU, + METRIC_DAU => $usersDAU, + METRIC_WAU => $usersWAU, METRIC_WEBHOOKS => $webhooks, METRIC_PLATFORMS => $platforms, METRIC_PROVIDERS => $providers, From 99f23ff25a6ff9ea305c41e39b8671d2fdf64a5f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:27:36 +0000 Subject: [PATCH 39/57] refactor --- .../Platform/Tasks/StatsResources.php | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index b2963ca87b..10e74a80e4 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -48,7 +48,7 @@ class StatsResources extends Action ->callback([$this, 'action']); } - public function action(Database $dbForPlatform, callable $logError, EventStatsResources $queueForStatsResources): void + public function action(Database $dbForPlatform, callable $logError, EventStatsResources $queue): void { $this->logError = $logError; $this->dbForPlatform = $dbForPlatform; @@ -58,35 +58,24 @@ class StatsResources extends Action Console::success('Stats resources: started'); $interval = (int) System::getEnv('_APP_STATS_RESOURCES_INTERVAL', '3600'); - Console::loop(function () use ($queueForStatsResources) { - $this->enqueueProjects($queueForStatsResources); + Console::loop(function () use ($queue) { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); + /** + * For each project that were accessed in last 24 hours + */ + $this->foreachDocument($this->dbForPlatform, 'projects', [ + Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours)) + ], function ($project) use ($queue) { + $queue + ->setProject($project) + ->trigger(); + Console::success('project: ' . $project->getId() . '(' . $project->getInternalId() . ')' . ' queued'); + }); }, $interval); Console::log("Stats resources: exited"); } - - /** - * Enqueue projects for counting - * @param Database $dbForPlatform - * @param EventStatsResources $queue - * @return void - */ - protected function enqueueProjects(EventStatsResources $queue): void - { - Authorization::disable(); - Authorization::setDefaultStatus(false); - - $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); - /** - * For each project that were accessed in last 24 hours - */ - $this->foreachDocument($this->dbForPlatform, 'projects', [ - Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours)) - ], function ($project) use ($queue) { - $queue - ->setProject($project) - ->trigger(); - Console::success('project: ' . $project->getId() . '(' . $project->getInternalId() . ')' . ' queued'); - }); - } } From 966fc47e2543bae97c48bd410ae02212310520da Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:28:44 +0000 Subject: [PATCH 40/57] fix classname --- src/Appwrite/Event/Event.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index f6638d2cf6..03ef7dd27f 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -33,7 +33,7 @@ class Event /** /remove */ public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources'; - public const STATS_RESOURCES_CLASS_NAME = 'StatsResources'; + public const STATS_RESOURCES_CLASS_NAME = 'StatsResourcesV1'; public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage'; public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1'; From 3e778c009437a21ce3c7408b9869d2fd4e8e74a1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:53:23 +0000 Subject: [PATCH 41/57] update environment variable --- .env | 2 +- CHANGES.md | 2 +- app/config/variables.php | 6 +++--- app/views/install/compose.phtml | 4 ++-- docker-compose.yml | 10 +++++----- .../Services/Functions/FunctionsCustomServerTest.php | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.env b/.env index 0b2f129f08..e52836f3a7 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=30 +_APP_STATS_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 diff --git a/CHANGES.md b/CHANGES.md index 62db3d525e..0e1be1698b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1687,7 +1687,7 @@ - Added new environment variables to enable error logging: - The `_APP_LOGGING_PROVIDER` variable allows you to enable the logger set the value to one of `sentry`, `raygun`, `appsignal`. - The `_APP_LOGGING_CONFIG` variable configures authentication to 3rd party error logging providers. If using Sentry, this should be 'SENTRY_API_KEY;SENTRY_APP_ID'. If using Raygun, this should be Raygun API key. If using AppSignal, this should be AppSignal API key. -- Added new environment variable `_APP_USAGE_AGGREGATION_INTERVAL` to configure the usage worker interval +- Added new environment variable `_APP_STATS_AGGREGATION_INTERVAL` to configure the usage worker interval - Added negative rotation values to file preview endpoint - Multiple responses from the Health service were changed to new (better) schema **Breaking Change** - Method `health.getAntiVirus()` has been renamed to `health.getAntivirus()` diff --git a/app/config/variables.php b/app/config/variables.php index 57810525c7..6603f246bc 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -224,7 +224,7 @@ return [ 'filter' => '' ], [ - 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', + 'name' => '_APP_STATS_AGGREGATION_INTERVAL', 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Database from TimeSeries data. The default value is 30 seconds. Reintroduced in 1.1.0.', 'introduction' => '1.1.0', 'default' => '30', @@ -234,7 +234,7 @@ return [ ], [ 'name' => '_APP_USAGE_TIMESERIES_INTERVAL', - 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', + 'description' => 'Deprecated since 1.1.0 use _APP_STATS_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '30', 'required' => false, @@ -243,7 +243,7 @@ return [ ], [ 'name' => '_APP_USAGE_DATABASE_INTERVAL', - 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', + 'description' => 'Deprecated since 1.1.0 use _APP_STATS_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '900', 'required' => false, diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index d8f3f649e0..1a1f07b8a0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -734,7 +734,7 @@ $image = $this->getParam('image', ''); - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_AGGREGATION_INTERVAL appwrite-worker-stats-usage-dump: image: /: @@ -762,7 +762,7 @@ $image = $this->getParam('image', ''); - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_AGGREGATION_INTERVAL appwrite-task-scheduler-functions: image: /: diff --git a/docker-compose.yml b/docker-compose.yml index 64ae108f7b..7e3be3383e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -781,7 +781,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES appwrite-worker-stats-usage: @@ -812,7 +812,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES appwrite-worker-stats-usage-dump: @@ -843,7 +843,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS @@ -876,7 +876,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES appwrite-worker-usage-dump: @@ -907,7 +907,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES # /remove diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index d8d1eb8eb5..9807e47999 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1687,7 +1687,7 @@ class FunctionsCustomServerTest extends Scope }); // Await Aggregation - sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30)); + sleep(System::getEnv('_APP_STATS_AGGREGATION_INTERVAL', 30)); $this->assertEventually(function () use ($functionId) { $response = $this->getFunctionUsage($functionId, [ From 68c3aa30784781e882ffc5581fc2da257ab5d72c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Feb 2025 09:53:29 +0000 Subject: [PATCH 42/57] update to use publisher --- src/Appwrite/Event/StatsResources.php | 17 +++-------------- src/Appwrite/Event/StatsUsage.php | 18 ++++++++---------- src/Appwrite/Event/StatsUsageDump.php | 6 +++--- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- src/Appwrite/Platform/Workers/Usage.php | 2 +- 5 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/Appwrite/Event/StatsResources.php b/src/Appwrite/Event/StatsResources.php index d08e1b2c8e..e7a3df97e0 100644 --- a/src/Appwrite/Event/StatsResources.php +++ b/src/Appwrite/Event/StatsResources.php @@ -2,13 +2,13 @@ namespace Appwrite\Event; -use Utopia\Queue\Connection; +use Utopia\Queue\Publisher; class StatsResources extends Event { - public function __construct(protected Connection $connection) + public function __construct(protected Publisher $publisher) { - parent::__construct($connection); + parent::__construct($publisher); $this ->setQueue(Event::STATS_RESOURCES_QUEUE_NAME) @@ -26,15 +26,4 @@ class StatsResources extends Event 'project' => $this->project ]; } - - /** - * Sends metrics to the usage worker. - * - * @return string|bool - */ - public function trigger(): string|bool - { - parent::trigger(); - return true; - } } diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php index a1c027f797..bed25419f6 100644 --- a/src/Appwrite/Event/StatsUsage.php +++ b/src/Appwrite/Event/StatsUsage.php @@ -3,17 +3,16 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; -use Utopia\Queue\Connection; +use Utopia\Queue\Publisher; class StatsUsage extends Event { protected array $metrics = []; protected array $reduce = []; - public function __construct(protected Connection $connection) + public function __construct(protected Publisher $publisher) { - parent::__construct($connection); + parent::__construct($publisher); $this ->setQueue(Event::STATS_USAGE_QUEUE_NAME) @@ -51,17 +50,16 @@ class StatsUsage extends Event } /** - * Sends metrics to the usage worker. + * Prepare the payload for the event * - * @return string|bool + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - $client = new Client($this->queue, $this->connection); - return $client->enqueue([ + return [ 'project' => $this->getProject(), 'reduce' => $this->reduce, 'metrics' => $this->metrics, - ]); + ]; } } diff --git a/src/Appwrite/Event/StatsUsageDump.php b/src/Appwrite/Event/StatsUsageDump.php index 3cd38eca92..0573a88040 100644 --- a/src/Appwrite/Event/StatsUsageDump.php +++ b/src/Appwrite/Event/StatsUsageDump.php @@ -2,15 +2,15 @@ namespace Appwrite\Event; -use Utopia\Queue\Connection; +use Utopia\Queue\Publisher; class StatsUsageDump extends Event { protected array $stats; - public function __construct(protected Connection $connection) + public function __construct(protected Publisher $publisher) { - parent::__construct($connection); + parent::__construct($publisher); $this ->setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 1bdf44d607..7289d01b44 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -58,7 +58,7 @@ class StatsUsage extends Action } //Todo Figure out way to preserve keys when the container is being recreated @shimonewman - $aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); + $aggregationInterval = (int) System::getEnv('_APP_STATS_AGGREGATION_INTERVAL', '20'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3687eeab67..1380d223df 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -41,7 +41,7 @@ class Usage extends Action $this->action($message, $project, $getProjectDB, $queueForUsageDump); }); - $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); + $this->aggregationInterval = (int) System::getEnv('_APP_STATS_AGGREGATION_INTERVAL', '20'); $this->lastTriggeredTime = time(); } From 87b3f535bdfc40a81c797b35738651d2b4b2c09d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 03:36:53 +0000 Subject: [PATCH 43/57] fix renaming --- app/controllers/api/health.php | 4 ++-- ...{get-queue-usage-dump.md => get-queue-stats-usage-dump.md} | 0 .../health/{get-queue-usage.md => get-queue-stats-usage.md} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename docs/references/health/{get-queue-usage-dump.md => get-queue-stats-usage-dump.md} (100%) rename docs/references/health/{get-queue-usage.md => get-queue-stats-usage.md} (100%) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 55e17e7cd6..213236cccc 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -692,7 +692,7 @@ App::get('/v1/health/queue/functions') $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); -App::get('/v1/health/queue/usage') +App::get('/v1/health/queue/stats-usage') ->desc('Get usage queue') ->groups(['api', 'health']) ->label('scope', 'health.read') @@ -700,7 +700,7 @@ App::get('/v1/health/queue/usage') auth: [AuthType::KEY], namespace: 'health', name: 'getQueueUsage', - description: '/docs/references/health/get-queue-usage.md', + description: '/docs/references/health/get-queue-stats-usage.md', responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, diff --git a/docs/references/health/get-queue-usage-dump.md b/docs/references/health/get-queue-stats-usage-dump.md similarity index 100% rename from docs/references/health/get-queue-usage-dump.md rename to docs/references/health/get-queue-stats-usage-dump.md diff --git a/docs/references/health/get-queue-usage.md b/docs/references/health/get-queue-stats-usage.md similarity index 100% rename from docs/references/health/get-queue-usage.md rename to docs/references/health/get-queue-stats-usage.md From 89aa61b17cdcca4759c94b2b7d51156dc7ee7aa2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 03:37:01 +0000 Subject: [PATCH 44/57] fix duplicate constant --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 6d3e2dd15e..70b95a117f 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -11,12 +11,12 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; -const METRIC_COLLECTION_LEVEL_STORAGE = 4; -const METRIC_DATABASE_LEVEL_STORAGE = 3; -const METRIC_PROJECT_LEVEL_STORAGE = 2; class StatsUsageDump extends Action { + const METRIC_COLLECTION_LEVEL_STORAGE = 4; + const METRIC_DATABASE_LEVEL_STORAGE = 3; + const METRIC_PROJECT_LEVEL_STORAGE = 2; protected array $stats = []; /** @@ -204,7 +204,7 @@ class StatsUsageDump extends Action switch (count($data)) { // Collection Level - case METRIC_COLLECTION_LEVEL_STORAGE: + case self::METRIC_COLLECTION_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); $databaseInternalId = $data[0]; $collectionInternalId = $data[1]; @@ -237,7 +237,7 @@ class StatsUsageDump extends Action $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); break; // Database Level - case METRIC_DATABASE_LEVEL_STORAGE: + case self::METRIC_DATABASE_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); $databaseInternalId = $data[0]; @@ -277,7 +277,7 @@ class StatsUsageDump extends Action $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); break; // Project Level - case METRIC_PROJECT_LEVEL_STORAGE: + case self::METRIC_PROJECT_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); // Get all project databases $databases = $dbForProject->find('database'); From 8da5bfccb689cd5b416d8874e58e8da05562c89a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 03:58:46 +0000 Subject: [PATCH 45/57] fix format and test --- src/Appwrite/Platform/Tasks/StatsResources.php | 2 +- src/Appwrite/Platform/Workers/StatsUsageDump.php | 7 +++---- tests/e2e/Services/Health/HealthCustomServerTest.php | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index 10e74a80e4..ac3b9ead73 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -61,7 +61,7 @@ class StatsResources extends Action Console::loop(function () use ($queue) { Authorization::disable(); Authorization::setDefaultStatus(false); - + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); /** * For each project that were accessed in last 24 hours diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 70b95a117f..eeb73dd378 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -11,12 +11,11 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; - class StatsUsageDump extends Action { - const METRIC_COLLECTION_LEVEL_STORAGE = 4; - const METRIC_DATABASE_LEVEL_STORAGE = 3; - const METRIC_PROJECT_LEVEL_STORAGE = 2; + public const METRIC_COLLECTION_LEVEL_STORAGE = 4; + public const METRIC_DATABASE_LEVEL_STORAGE = 3; + public const METRIC_PROJECT_LEVEL_STORAGE = 2; protected array $stats = []; /** diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 42039c7e10..831916e043 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -499,7 +499,7 @@ class HealthCustomServerTest extends Scope /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -511,7 +511,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/usage?threshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); From c3495dde358c617304a22e04f20607ca1ae77559 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 04:17:49 +0000 Subject: [PATCH 46/57] fix key --- src/Appwrite/Platform/Workers/StatsUsage.php | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 7289d01b44..68ef966f07 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -21,7 +21,7 @@ class StatsUsage extends Action public static function getName(): string { - return 'usage'; + return 'stats-usage'; } /** @@ -31,13 +31,11 @@ class StatsUsage extends Action { $this - ->desc('Usage worker') - ->inject('message') - ->inject('getProjectDB') - ->inject('queueForStatsUsageDump') - ->callback(function (Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump) { - $this->action($message, $getProjectDB, $queueForStatsUsageDump); - }); + ->desc('Stats usage worker') + ->inject('message') + ->inject('getProjectDB') + ->inject('queueForStatsUsageDump') + ->callback([$this, 'action']); $this->lastTriggeredTime = time(); } From 2efe1ed9dc78e044c6df1a7e3867e52643db40ab Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 04:24:04 +0000 Subject: [PATCH 47/57] new health endpoint --- app/controllers/api/health.php | 34 ++++++++++++++++++- .../health/get-queue-stats-resources.md | 1 + 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 docs/references/health/get-queue-stats-resources.md diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 213236cccc..aae1faffee 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -692,8 +692,40 @@ App::get('/v1/health/queue/functions') $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); +App::get('/v1/health/queue/stats-resources') + ->desc('Get stats resources queue') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk', new Method( + auth: [AuthType::KEY], + namespace: 'health', + name: 'getQueueStatsResources', + description: '/docs/references/health/get-queue-stats-resources.md', + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_HEALTH_QUEUE, + ) + ], + contentType: ContentType::JSON + )) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->inject('publisher') + ->inject('response') + ->action(function (int|string $threshold, Publisher $publisher, Response $response) { + $threshold = \intval($threshold); + + $size = $publisher->getQueueSize(new Queue(Event::STATS_USAGE_QUEUE_NAME)); + + if ($size >= $threshold) { + throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); + }); + App::get('/v1/health/queue/stats-usage') - ->desc('Get usage queue') + ->desc('Get stats usage queue') ->groups(['api', 'health']) ->label('scope', 'health.read') ->label('sdk', new Method( diff --git a/docs/references/health/get-queue-stats-resources.md b/docs/references/health/get-queue-stats-resources.md new file mode 100644 index 0000000000..5221327467 --- /dev/null +++ b/docs/references/health/get-queue-stats-resources.md @@ -0,0 +1 @@ +Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue. \ No newline at end of file From ae9d3f4703dc79b4295d84220612b48cf347b877 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 04:40:01 +0000 Subject: [PATCH 48/57] fix dual writing --- .env | 2 +- src/Appwrite/Platform/Workers/StatsUsageDump.php | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.env b/.env index e52836f3a7..2e4b5fa5c7 100644 --- a/.env +++ b/.env @@ -111,4 +111,4 @@ _APP_MESSAGE_PUSH_TEST_DSN= _APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 _APP_PROJECT_REGIONS=default _APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000 -_APP_STATS_USAGE_DUAL_WRITING_DBS= \ No newline at end of file +_APP_STATS_USAGE_DUAL_WRITING_DBS=database_db_main \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index eeb73dd378..1b5ec10151 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -150,13 +150,14 @@ class StatsUsageDump extends Action 'value' => $value, 'region' => System::getEnv('_APP_REGION', 'default'), ]); + $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( 'stats', 'value', [$document] ); - $this->writeToLogsDB($project, $document); + $this->writeToLogsDB($project, $documentClone); } } } catch (\Exception $e) { @@ -181,12 +182,13 @@ class StatsUsageDump extends Action 'value' => $value, 'region' => System::getEnv('_APP_REGION', 'default'), ]); + $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( 'stats', 'value', [$document] ); - $this->writeToLogsDB($project, $document); + $this->writeToLogsDB($project, $documentClone); }; foreach ($this->periods as $period => $format) { @@ -337,5 +339,6 @@ class StatsUsageDump extends Action 'value', [$document] ); + Console::success('Usage logs pushed to Logs DB'); } } From c27c3499aa420dd8e33392ab4876992ddb853f0a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 04:48:40 +0000 Subject: [PATCH 49/57] fix endpoint and add test --- app/controllers/api/health.php | 2 +- .../Health/HealthCustomServerTest.php | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index aae1faffee..1551b7aca7 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -715,7 +715,7 @@ App::get('/v1/health/queue/stats-resources') ->action(function (int|string $threshold, Publisher $publisher, Response $response) { $threshold = \intval($threshold); - $size = $publisher->getQueueSize(new Queue(Event::STATS_USAGE_QUEUE_NAME)); + $size = $publisher->getQueueSize(new Queue(Event::STATS_RESOURCES_QUEUE_NAME)); if ($size >= $threshold) { throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 831916e043..04b1408cd0 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -494,6 +494,30 @@ class HealthCustomServerTest extends Scope return []; } + public function testStatsResources() + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-resources', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsInt($response['body']['size']); + $this->assertLessThan(100, $response['body']['size']); + + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-resources?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(503, $response['headers']['status-code']); + } + public function testUsageSuccess() { /** From c48146b699f0aebf95938b8c9b0d98321044f671 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 06:05:53 +0000 Subject: [PATCH 50/57] fix builds worker --- src/Appwrite/Platform/Workers/Builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index f6e06ac0e5..e7cbbd5088 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -55,7 +55,7 @@ class Builds extends Action ->inject('dbForProject') ->inject('deviceForFunctions') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); } /** From f6b2d9d937faa670a775778e0c3ac8e01a7e60b7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 07:02:07 +0000 Subject: [PATCH 51/57] remove old collections - we only need it in production temporarily --- docker-compose.yml | 64 ---------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7e3be3383e..2603e707e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -847,70 +847,6 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS - # remove - appwrite-worker-usage: - entrypoint: worker-usage - <<: *x-logging - container_name: appwrite-worker-usage - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_USAGE_STATS - - _APP_LOGGING_CONFIG - - _APP_STATS_AGGREGATION_INTERVAL - - _APP_DATABASE_SHARED_TABLES - - appwrite-worker-usage-dump: - entrypoint: worker-usage-dump - <<: *x-logging - container_name: appwrite-worker-usage-dump - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_USAGE_STATS - - _APP_LOGGING_CONFIG - - _APP_STATS_AGGREGATION_INTERVAL - - _APP_DATABASE_SHARED_TABLES - # /remove - appwrite-task-scheduler-functions: entrypoint: schedule-functions <<: *x-logging From cd4517755851a38313a9c7a790d276707d88c51e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 6 Feb 2025 08:06:15 +0000 Subject: [PATCH 52/57] fix pool empty --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- .../Platform/Workers/StatsUsageDump.php | 31 +++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 68ef966f07..f0f7d99b03 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -43,7 +43,7 @@ class StatsUsage extends Action /** * @param Message $message * @param callable $getProjectDB - * @param StatsUsageDump $queueForUsageDump + * @param StatsUsageDump $queueForStatsUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 1b5ec10151..a1b776b720 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -9,6 +9,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Registry\Registry; use Utopia\System\System; class StatsUsageDump extends Action @@ -18,6 +19,8 @@ class StatsUsageDump extends Action public const METRIC_PROJECT_LEVEL_STORAGE = 2; protected array $stats = []; + protected Registry $register; + /** * Metrics to skip writing to logsDB * As these metrics are calculated separately @@ -87,6 +90,7 @@ class StatsUsageDump extends Action ->inject('message') ->inject('getProjectDB') ->inject('getLogsDB') + ->inject('register') ->callback([$this, 'action']); } @@ -98,9 +102,10 @@ class StatsUsageDump extends Action * @throws Exception * @throws \Utopia\Database\Exception */ - public function action(Message $message, callable $getProjectDB, callable $getLogsDB): void + public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void { $this->getLogsDB = $getLogsDB; + $this->register = $register; $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); @@ -322,9 +327,6 @@ class StatsUsageDump extends Action return; } - /** @var \Utopia\Database\Database $dbForLogs*/ - $dbForLogs = call_user_func($this->getLogsDB, $project); - if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) { return; } @@ -334,11 +336,20 @@ class StatsUsageDump extends Action } } - $dbForLogs->createOrUpdateDocumentsWithIncrease( - 'stats', - 'value', - [$document] - ); - Console::success('Usage logs pushed to Logs DB'); + /** @var \Utopia\Database\Database $dbForLogs*/ + $dbForLogs = call_user_func($this->getLogsDB, $project); + + try { + $dbForLogs->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + [$document] + ); + Console::success('Usage logs pushed to Logs DB'); + } catch (\Throwable $th) { + Console::error($th->getMessage()); + } + + $this->register->get('pools')->get('logs')->reclaim(); } } From 59516408af5d0c5a9a8dbca0810f84d13ee12fc0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 9 Feb 2025 05:15:26 +0000 Subject: [PATCH 53/57] revert env update --- .env | 2 +- CHANGES.md | 2 +- app/config/variables.php | 6 +++--- app/views/install/compose.phtml | 4 ++-- docker-compose.yml | 6 +++--- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- src/Appwrite/Platform/Workers/Usage.php | 2 +- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.env b/.env index 2e4b5fa5c7..06a6fa9b3e 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_STATS_AGGREGATION_INTERVAL=30 +_APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 diff --git a/CHANGES.md b/CHANGES.md index 0e1be1698b..62db3d525e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1687,7 +1687,7 @@ - Added new environment variables to enable error logging: - The `_APP_LOGGING_PROVIDER` variable allows you to enable the logger set the value to one of `sentry`, `raygun`, `appsignal`. - The `_APP_LOGGING_CONFIG` variable configures authentication to 3rd party error logging providers. If using Sentry, this should be 'SENTRY_API_KEY;SENTRY_APP_ID'. If using Raygun, this should be Raygun API key. If using AppSignal, this should be AppSignal API key. -- Added new environment variable `_APP_STATS_AGGREGATION_INTERVAL` to configure the usage worker interval +- Added new environment variable `_APP_USAGE_AGGREGATION_INTERVAL` to configure the usage worker interval - Added negative rotation values to file preview endpoint - Multiple responses from the Health service were changed to new (better) schema **Breaking Change** - Method `health.getAntiVirus()` has been renamed to `health.getAntivirus()` diff --git a/app/config/variables.php b/app/config/variables.php index 6603f246bc..57810525c7 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -224,7 +224,7 @@ return [ 'filter' => '' ], [ - 'name' => '_APP_STATS_AGGREGATION_INTERVAL', + 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Database from TimeSeries data. The default value is 30 seconds. Reintroduced in 1.1.0.', 'introduction' => '1.1.0', 'default' => '30', @@ -234,7 +234,7 @@ return [ ], [ 'name' => '_APP_USAGE_TIMESERIES_INTERVAL', - 'description' => 'Deprecated since 1.1.0 use _APP_STATS_AGGREGATION_INTERVAL instead.', + 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '30', 'required' => false, @@ -243,7 +243,7 @@ return [ ], [ 'name' => '_APP_USAGE_DATABASE_INTERVAL', - 'description' => 'Deprecated since 1.1.0 use _APP_STATS_AGGREGATION_INTERVAL instead.', + 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '900', 'required' => false, diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 1a1f07b8a0..d8f3f649e0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -734,7 +734,7 @@ $image = $this->getParam('image', ''); - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_STATS_AGGREGATION_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL appwrite-worker-stats-usage-dump: image: /: @@ -762,7 +762,7 @@ $image = $this->getParam('image', ''); - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_STATS_AGGREGATION_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL appwrite-task-scheduler-functions: image: /: diff --git a/docker-compose.yml b/docker-compose.yml index 2603e707e8..5391bbe397 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -781,7 +781,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_STATS_AGGREGATION_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES appwrite-worker-stats-usage: @@ -812,7 +812,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_STATS_AGGREGATION_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES appwrite-worker-stats-usage-dump: @@ -843,7 +843,7 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_STATS_AGGREGATION_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index f0f7d99b03..c4d8b0e8d2 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -56,7 +56,7 @@ class StatsUsage extends Action } //Todo Figure out way to preserve keys when the container is being recreated @shimonewman - $aggregationInterval = (int) System::getEnv('_APP_STATS_AGGREGATION_INTERVAL', '20'); + $aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 1380d223df..3687eeab67 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -41,7 +41,7 @@ class Usage extends Action $this->action($message, $project, $getProjectDB, $queueForUsageDump); }); - $this->aggregationInterval = (int) System::getEnv('_APP_STATS_AGGREGATION_INTERVAL', '20'); + $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $this->lastTriggeredTime = time(); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 9807e47999..d8d1eb8eb5 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1687,7 +1687,7 @@ class FunctionsCustomServerTest extends Scope }); // Await Aggregation - sleep(System::getEnv('_APP_STATS_AGGREGATION_INTERVAL', 30)); + sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30)); $this->assertEventually(function () use ($functionId) { $response = $this->getFunctionUsage($functionId, [ From 55e30a0ef49a9af942e637dcc9e6433ec5495fdf Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 9 Feb 2025 12:19:16 +0545 Subject: [PATCH 54/57] Update src/Appwrite/Platform/Workers/StatsUsageDump.php Co-authored-by: Christy Jacob --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index a1b776b720..581bacdc6c 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -115,9 +115,6 @@ class StatsUsageDump extends Action foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); - /** - * End temp bug fallback - */ $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; $receivedAt = $stats['receivedAt'] ?? 'NONE'; if ($numberOfKeys === 0) { From ec9045b488c935ed31b441884c888d420fe8d628 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 9 Feb 2025 06:36:59 +0000 Subject: [PATCH 55/57] add stats-resources health queue --- app/controllers/api/health.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 1551b7aca7..e5336067c8 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -952,6 +952,7 @@ App::get('/v1/health/queue/failed/:name') Event::AUDITS_QUEUE_NAME, Event::MAILS_QUEUE_NAME, Event::FUNCTIONS_QUEUE_NAME, + Event::STATS_RESOURCES_QUEUE_NAME, Event::STATS_USAGE_QUEUE_NAME, Event::STATS_USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_QUEUE_NAME, From e8faf2a85939ddb9ffeb03b91432103d87bca0f0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 9 Feb 2025 08:52:07 +0000 Subject: [PATCH 56/57] fix worker --- src/Appwrite/Platform/Workers/StatsResources.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 668d71d703..e3c76ecb9a 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -85,11 +85,20 @@ class StatsResources extends Action { Console::info('Begining count for: ' . $project->getId()); + $dbForLogs = null; + $dbForProject = null; try { /** @var \Utopia\Database\Database $dbForLogs */ $dbForLogs = call_user_func($getLogsDB, $project); /** @var \Utopia\Database\Database $dbForProject */ $dbForProject = call_user_func($getProjectDB, $project); + } catch (Throwable $th) { + Console::error('Unable to get database'); + Console::error($th->getMessage()); + return; + } + + try { $region = $project->getAttribute('region'); @@ -162,12 +171,12 @@ class StatsResources extends Action } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]); } + + $this->writeDocuments($dbForLogs, $project); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); } - $this->writeDocuments($dbForLogs, $project); - Console::info('End of count for: ' . $project->getId()); } From bfc0b531630b721992b8024d71b3da96a9bb639b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 10 Feb 2025 23:05:34 +0530 Subject: [PATCH 57/57] chore: add simple toggle for dual writing --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 581bacdc6c..38ebd578a5 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -317,10 +317,8 @@ class StatsUsageDump extends Action protected function writeToLogsDB(Document $project, Document $document) { - $databasesToDualWrite = explode(',', System::getEnv('_APP_STATS_USAGE_DUAL_WRITING_DBS', '')); - - $db = $project->getAttribute('database'); - if (!in_array($db, $databasesToDualWrite)) { + if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) { + Console::log('Dual Writing is disabled. Skipping...'); return; }