From 7ea8d761e475603a97d9b7340c53586ae3a6e91c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 21 Dec 2022 02:08:37 +0530 Subject: [PATCH] feat: aggregate stats --- Dockerfile | 1 + bin/stat | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/Stat.php | 212 +++++++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 bin/stat create mode 100644 src/Appwrite/Platform/Tasks/Stat.php diff --git a/Dockerfile b/Dockerfile index 54aa327769..9a8e35f730 100755 --- a/Dockerfile +++ b/Dockerfile @@ -314,6 +314,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ + chmod +x /usr/local/bin/stat && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/vars && \ chmod +x /usr/local/bin/worker-audits && \ diff --git a/bin/stat b/bin/stat new file mode 100644 index 0000000000..d7030a0958 --- /dev/null +++ b/bin/stat @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php stat $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2e15cd015c..2cb0878589 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -12,6 +12,7 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; +use Appwrite\Platform\Tasks\Stat; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; @@ -27,6 +28,7 @@ class Tasks extends Service ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) + ->addAction(Stat::getName(), new Stat()) ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) diff --git a/src/Appwrite/Platform/Tasks/Stat.php b/src/Appwrite/Platform/Tasks/Stat.php new file mode 100644 index 0000000000..8ac5687bae --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Stat.php @@ -0,0 +1,212 @@ +desc('Get stats for project') + ->callback(fn () => $this->action()); + } + + function getConnection(string $dsn): PDO + { + if (empty($dsn)) { + throw new Exception("Missing value for DSN connection"); + } + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getPath(); + + $connection = new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_WARNING, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + + return $connection; + } + + + function getStats(Database $dbForProject): array + { + $range = '90d'; + $periods = [ + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + 'files.$all.count.total', + 'buckets.$all.count.total', + 'databases.$all.count.total', + 'documents.$all.count.total', + 'collections.$all.count.total', + 'project.$all.storage.size', + 'project.$all.network.requests', + 'project.$all.network.bandwidth', + 'users.$all.count.total', + 'sessions.$all.requests.create', + 'executions.$all.compute.total', + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + // Calculate aggregate of each metric + $stats[$metric . '.sum'] = array_sum(array_column($stats[$metric], 'value')); + } + }); + + // return only the ahhggregate values + return array_filter($stats, fn ($key) => strpos($key, '.sum') !== false, ARRAY_FILTER_USE_KEY); + } + + + public function action(): void + { + Console::success('Getting stats...'); + + $databases = [ + 'console' => [ + 'type' => 'database', + 'dsns' => '', + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'projects' => [ + 'type' => 'database', + 'dsns' => '', + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + ]; + + $dsns = explode(',', $databases['projects']['dsns']); + $projectdsns = []; + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = 'database' . '_' . $dsn[0]; + $dsn = $dsn[1] ?? ''; + $projectdsns[$name] = $dsn; + } + + $cache = new Cache(new None()); + $consoledsn = explode('=', $databases['console']['dsns']); + $consoledsn = $consoledsn[1] ?? ''; + $adapter = new MySQL($this->getConnection($consoledsn)); + $dbForConsole = new Database($adapter, $cache); + $dbForConsole->setDefaultDatabase('appwrite'); + $dbForConsole->setNamespace('console'); + + $totalProjects = $dbForConsole->count('projects') + 1; + Console::success("Iterating through : {$totalProjects} projects"); + + $app = new App('UTC'); + $console = $app->getResource('console'); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + + $stats = []; + + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + Console::info("Getting stats for {$project->getId()}"); + + try { + // TODO: Iterate through all project DBs + $db = $project->getAttribute('database'); + $dsn = $projectdsns[$db] ?? ''; + $cache = new Cache(new None()); + $adapter = new MySQL($this->getConnection($dsn)); + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + $statsPerProject = $this->getStats($dbForProject); + + foreach ($statsPerProject as $key => $value) { + $stats[$key] = isset($stats[$key]) ? $stats[$key] + $value : $value; + } + + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } + + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + + Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); + } + + var_dump($stats); + } +}