From dea3e74b6adb0f983a00d08e5a026d7dad50477f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 28 Nov 2023 10:19:55 +0000 Subject: [PATCH 01/10] Implement Job based hamster --- Dockerfile | 3 +- app/cli.php | 4 + app/init.php | 4 + app/worker.php | 4 + bin/worker-hamster | 3 + docker-compose.yml | 31 ++ src/Appwrite/Event/Event.php | 3 + src/Appwrite/Event/Hamster.php | 153 +++++++ src/Appwrite/Platform/Services/Workers.php | 2 + src/Appwrite/Platform/Tasks/Hamster.php | 373 +++-------------- src/Appwrite/Platform/Workers/Hamster.php | 454 +++++++++++++++++++++ 11 files changed, 708 insertions(+), 326 deletions(-) create mode 100644 bin/worker-hamster create mode 100644 src/Appwrite/Event/Hamster.php create mode 100644 src/Appwrite/Platform/Workers/Hamster.php diff --git a/Dockerfile b/Dockerfile index 059c499bd9..599e4ea70e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats + chmod +x /usr/local/bin/calc-tier-stats && \ + chmod +x /usr/local/bin/worker-hamster # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/cli.php b/app/cli.php index 643a615c46..003f3a1f79 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,6 +6,7 @@ require_once __DIR__ . '/controllers/general.php'; use Appwrite\Event\Delete; use Appwrite\Event\Certificate; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; @@ -154,6 +155,9 @@ CLI::setResource('queue', function (Group $pools) { CLI::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); +CLI::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); CLI::setResource('queueForDeletes', function (Connection $queue) { return new Delete($queue); }, ['queue']); diff --git a/app/init.php b/app/init.php index 2c0219eec2..c30eb77e85 100644 --- a/app/init.php +++ b/app/init.php @@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException; use Appwrite\Event\Build; use Appwrite\Event\Certificate; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; @@ -916,6 +917,9 @@ App::setResource('queueForCertificates', function (Connection $queue) { App::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); +App::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); diff --git a/app/worker.php b/app/worker.php index 32a8b9804e..4f7355311e 100644 --- a/app/worker.php +++ b/app/worker.php @@ -9,6 +9,7 @@ use Appwrite\Event\Certificate; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; @@ -155,6 +156,9 @@ Server::setResource('queueForCertificates', function (Connection $queue) { Server::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); +Server::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); Server::setResource('logger', function (Registry $register) { return $register->get('logger'); }, ['register']); diff --git a/bin/worker-hamster b/bin/worker-hamster new file mode 100644 index 0000000000..b388dd13c9 --- /dev/null +++ b/bin/worker-hamster @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php hamster $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 42091e5e46..da9b0e51e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -717,6 +717,37 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-worker-hamster: + entrypoint: worker-hamster + <<: *x-logging + container_name: appwrite-worker-hamster + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_LOGGING_PROVIDER + - _APP_MIXPANEL_TOKEN + openruntimes-executor: container_name: openruntimes-executor hostname: appwrite-executor diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 46b430d122..fc12c5b5b3 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -42,6 +42,9 @@ class Event public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; + public const HAMSTER_QUEUE_NAME = 'v1-hamster'; + public const HAMSTER_CLASS_NAME = 'HamsterV1'; + protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php new file mode 100644 index 0000000000..9ae7303674 --- /dev/null +++ b/src/Appwrite/Event/Hamster.php @@ -0,0 +1,153 @@ +setQueue(Event::HAMSTER_QUEUE_NAME) + ->setClass(Event::HAMSTER_CLASS_NAME); + } + + /** + * Sets the type for the hamster event. + * + * @param string $type + * @return self + */ + public function setType(string $type): self + { + $this->type = $type; + + return $this; + } + + /** + * Returns the set type for the hamster event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + + /** + * Sets the project for the hamster event. + * + * @param Document $project + */ + public function setProject(Document $project): self + { + $this->project = $project; + + return $this; + } + + /** + * Returns the set project for the hamster event. + * + * @return Document + */ + public function getProject(): Document + { + return $this->project; + } + + /** + * Sets the organization for the hamster event. + * + * @param Document $organization + */ + public function setOrganization(Document $organization): self + { + $this->organization = $organization; + + return $this; + } + + /** + * Returns the set organization for the hamster event. + * + * @return string + */ + public function getOrganization(): Document + { + return $this->organization; + } + + /** + * Sets the user for the hamster event. + * + * @param Document $user + */ + public function setUser(Document $user): self + { + $this->user = $user; + + return $this; + } + + /** + * Returns the set user for the hamster event. + * + * @return Document + */ + public function getUser(): Document + { + return $this->user; + } + + /** + * Executes the function event and sends it to the functions worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + if ($this->paused) { + return false; + } + + $client = new Client($this->queue, $this->connection); + + $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; + + return $client->enqueue([ + 'type' => $this->type, + 'project' => $this->project, + 'organization' => $this->organization, + 'user' => $this->user, + 'events' => $events, + ]); + } + + /** + * Generate a function event from a base event + * + * @param Event $event + * + * @return self + * + */ + public function from(Event $event): self + { + $this->event = $event->getEvent(); + $this->params = $event->getParams(); + return $this; + } +} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 07fc25434e..c5a0514760 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -12,6 +12,7 @@ use Appwrite\Platform\Workers\Databases; use Appwrite\Platform\Workers\Functions; use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; +use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -30,6 +31,7 @@ class Workers extends Service ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) ->addAction(Migrations::getName(), new Migrations()) + ->addAction(Hamster::getName(), new Hamster()) ; } diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 1d5d3b0b26..fea4c88ade 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Tasks; +use Appwrite\Event\Hamster as EventHamster; use Appwrite\Network\Validator\Origin; use Exception; use Utopia\App; @@ -19,20 +20,6 @@ use Utopia\Pools\Group; class Hamster extends Action { - private array $metrics = [ - 'usage_files' => 'files.$all.count.total', - 'usage_buckets' => 'buckets.$all.count.total', - 'usage_databases' => 'databases.$all.count.total', - 'usage_documents' => 'documents.$all.count.total', - 'usage_collections' => 'collections.$all.count.total', - 'usage_storage' => 'project.$all.storage.size', - 'usage_requests' => 'project.$all.network.requests', - 'usage_bandwidth' => 'project.$all.network.bandwidth', - 'usage_users' => 'users.$all.count.total', - 'usage_sessions' => 'sessions.email.requests.create', - 'usage_executions' => 'executions.$all.compute.total', - ]; - protected string $directory = '/usr/local'; protected string $path; @@ -60,233 +47,20 @@ class Hamster extends Action }); } - private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole) + private function getStatsPerProject(Group $pools, Database $dbForConsole) { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - Console::info("Skipping project console"); - return; - } + $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools) { + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); - Console::log("Getting stats for {$project->getId()}"); + $hamsterTask = new EventHamster($connection); - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); + $hamsterTask + ->setType('project') + ->setProject($project) + ->trigger(); - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $statsPerProject = []; - - $statsPerProject['time'] = microtime(true); - - /** Get Project ID */ - $statsPerProject['project_id'] = $project->getId(); - - /** Get project created time */ - $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); - - /** Get Project Name */ - $statsPerProject['project_name'] = $project->getAttribute('name'); - - /** Total Project Variables */ - $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); - - /** Total Migrations */ - $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); - - /** Get Custom SMTP */ - $smtp = $project->getAttribute('smtp', null); - if ($smtp) { - $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; - - /** Get Custom Templates Count */ - $templates = array_keys($project->getAttribute('templates', [])); - $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'email'); - }); - $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'sms'); - }); - } - - /** Get total relationship attributes */ - $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ - Query::equal('type', ['relationship']) - ], APP_LIMIT_COUNT); - - /** Get Total Functions */ - $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - - foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { - $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ - Query::equal('runtime', [$runtime]), - ], APP_LIMIT_COUNT); - } - - /** Get Total Deployments */ - $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ - Query::equal('type', ['manual']) - ], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ - Query::equal('type', ['vcs']) - ], APP_LIMIT_COUNT); - - /** Get VCS repos connected */ - $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ], APP_LIMIT_COUNT); - - /** Get Total Teams */ - $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ], APP_LIMIT_COUNT); - } else { - $statsPerProject['custom_organization_members'] = 0; - } - - /** Get Email and Name of the project owner */ - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new Exception('Membership not found. Skipping project : ' . $project->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerProject['email'] = $user->getAttribute('email', null); - $statsPerProject['name'] = $user->getAttribute('name', null); - } - } - - /** Get Domains */ - $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Platforms */ - $platforms = $dbForConsole->find('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'web'; - })); - - $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'android'; - })); - - $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'apple'); - })); - - $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'flutter'); - })); - - $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; - - foreach ($flutterPlatforms as $flutterPlatform) { - $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { - return $platform['type'] === $flutterPlatform; - })); - } - - $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Usage $statsPerProject */ - $periods = [ - 'infinity' => [ - 'period' => '1d', - 'limit' => 90, - ], - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - ]; - - Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { - foreach ($this->metrics as $key => $metric) { - foreach ($periods as $periodKey => $periodValue) { - $limit = $periodValue['limit']; - $period = $periodValue['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $statsPerProject[$key . '_' . $periodKey] = []; - foreach ($requestDocs as $requestDoc) { - $statsPerProject[$key . '_' . $periodKey][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); - // Calculate aggregate of each metric - $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); - } - } - }); - - if (isset($statsPerProject['email'])) { - /** Send data to mixpanel */ - $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ - 'name' => $statsPerProject['name'], - 'email' => $statsPerProject['email'] - ]); - - if (!$res) { - Console::error('Failed to create user profile for project: ' . $project->getId()); - } - } - - $event = new Event(); - $event - ->setName('Project Daily Usage') - ->setProps($statsPerProject); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - Console::error('Failed to create event for project: ' . $project->getId()); - } - } catch (Exception $e) { - Console::error('Failed to send stats for project: ' . $project->getId()); - Console::error($e->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } + $queue->reclaim(); }); } @@ -305,6 +79,8 @@ class Hamster extends Action $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); $delay = $next->getTimestamp() - $now->getTimestamp(); + $delay = 5; + /** * If time passed for the target day. */ @@ -323,17 +99,17 @@ class Hamster extends Action /* Initialise new Utopia app */ $app = new App('UTC'); - Console::info('Getting stats for all projects'); - $this->getStatsPerProject($pools, $cache, $dbForConsole); - Console::success('Completed getting stats for all projects'); + Console::info('Queuing stats for all projects'); + $this->getStatsPerProject($pools, $dbForConsole); + Console::success('Completed queuing stats for all projects'); - Console::info('Getting stats for all organizations'); - $this->getStatsPerOrganization($dbForConsole); - Console::success('Completed getting stats for all organizations'); + Console::info('Queuing stats for all organizations'); + $this->getStatsPerOrganization($pools, $dbForConsole); + Console::success('Completed queuing stats for all organizations'); - Console::info('Getting stats for all users'); - $this->getStatsPerUser($dbForConsole); - Console::success('Completed getting stats for all users'); + Console::info('Queuing stats for all users'); + $this->getStatsPerUser($pools, $dbForConsole); + Console::success('Completed queuing stats for all users'); $pools ->get('console') @@ -378,96 +154,43 @@ class Hamster extends Action Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } - protected function getStatsPerOrganization(Database $dbForConsole) + protected function getStatsPerOrganization(Group $pools, Database $dbForConsole) { - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) { + $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($pools) { try { - $statsPerOrganization = []; - - /** Organization name */ - $statsPerOrganization['name'] = $document->getAttribute('name'); - - /** Get Email and of the organization owner */ - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$document->getInternalId()]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new Exception('Membership not found. Skipping organization : ' . $document->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerOrganization['email'] = $user->getAttribute('email', null); - } - - /** Organization Creation Date */ - $statsPerOrganization['created'] = $document->getAttribute('$createdAt'); - - /** Number of team members */ - $statsPerOrganization['members'] = $document->getAttribute('total'); - - /** Number of projects in this organization */ - $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ - Query::equal('teamId', [$document->getId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerOrganization['email'])) { - throw new Exception('Email not found. Skipping organization : ' . $document->getId()); - } - - $event = new Event(); - $event - ->setName('Organization Daily Usage') - ->setProps($statsPerOrganization); - $res = $this->mixpanel->createEvent($event); - if (!$res) { - throw new Exception('Failed to create event for organization : ' . $document->getId()); - } + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + + $hamsterTask = new EventHamster($connection); + + $hamsterTask + ->setType('organization') + ->setOrganization($organization) + ->trigger(); + + $queue->reclaim(); } catch (Exception $e) { Console::error($e->getMessage()); } }); } - protected function getStatsPerUser(Database $dbForConsole) + protected function getStatsPerUser(Group $pools, Database $dbForConsole) { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) { + $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($pools) { try { - $statsPerUser = []; - - /** Organization name */ - $statsPerUser['name'] = $document->getAttribute('name'); - - /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ - $statsPerUser['email'] = $document->getAttribute('email'); - - /** Organization Creation Date */ - $statsPerUser['created'] = $document->getAttribute('$createdAt'); - - /** Number of teams this user is a part of */ - $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerUser['email'])) { - throw new Exception('User has no email: ' . $document->getId()); - } - - /** Send data to mixpanel */ - $event = new Event(); - $event - ->setName('User Daily Usage') - ->setProps($statsPerUser); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - throw new Exception('Failed to create user profile for user: ' . $document->getId()); - } + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + + $hamsterTask = new EventHamster($connection); + + $hamsterTask + ->setType('user') + ->setUser($user) + ->trigger(); + + $queue->reclaim(); } catch (Exception $e) { Console::error($e->getMessage()); } diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php new file mode 100644 index 0000000000..d1c03744c8 --- /dev/null +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -0,0 +1,454 @@ + 'files.$all.count.total', + 'usage_buckets' => 'buckets.$all.count.total', + 'usage_databases' => 'databases.$all.count.total', + 'usage_documents' => 'documents.$all.count.total', + 'usage_collections' => 'collections.$all.count.total', + 'usage_storage' => 'project.$all.storage.size', + 'usage_requests' => 'project.$all.network.requests', + 'usage_bandwidth' => 'project.$all.network.bandwidth', + 'usage_users' => 'users.$all.count.total', + 'usage_sessions' => 'sessions.email.requests.create', + 'usage_executions' => 'executions.$all.compute.total', + ]; + + protected string $directory = '/usr/local'; + + protected string $path; + + protected string $date; + + protected Mixpanel $mixpanel; + + public static function getName(): string + { + return 'hamster'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', '')); + + $this + ->desc('Hamster worker') + ->inject('message') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('queueForHamster') + ->inject('queueForEvents') + ->inject('usage') + ->inject('log') + ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $pools, $cache, $dbForConsole, $queueForHamster, $queueForEvents, $usage, $log)); + } + + /** + * @param Message $message + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * @param EventHamster $queueForHamster + * @param Event $queueForEvents + * @param Stats $usage + * @param Log $log + * @return void + * @throws Authorization + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Conflict + */ + public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + + switch ($type) { + case 'project': + $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole, $log); + break; + case 'organization': + $this->getStatsForOrganization(new Document($payload['organization']), $pools, $cache, $dbForConsole, $log); + break; + case 'user': + $this->getStatsPerUser(new Document($payload['user']), $dbForConsole); + break; + } + } + + /** + * @param Document $project + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * @param Log $log + * @throws \Utopia\Database\Exception + */ + private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void + { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + Console::info("Skipping project console"); + return; + } + + Console::log("Getting stats for {$project->getId()}"); + + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = []; + + $statsPerProject['time'] = microtime(true); + + /** Get Project ID */ + $statsPerProject['project_id'] = $project->getId(); + + /** Get project created time */ + $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); + + /** Get Project Name */ + $statsPerProject['project_name'] = $project->getAttribute('name'); + + /** Total Project Variables */ + $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); + + /** Total Migrations */ + $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); + + /** Get Custom SMTP */ + $smtp = $project->getAttribute('smtp', null); + if ($smtp) { + $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; + + /** Get Custom Templates Count */ + $templates = array_keys($project->getAttribute('templates', [])); + $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { + return str_contains($template, 'email'); + }); + $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { + return str_contains($template, 'sms'); + }); + } + + /** Get total relationship attributes */ + $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ + Query::equal('type', ['relationship']) + ], APP_LIMIT_COUNT); + + /** Get Total Functions */ + $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); + + foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { + $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ + Query::equal('runtime', [$runtime]), + ], APP_LIMIT_COUNT); + } + + /** Get Total Deployments */ + $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); + $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ + Query::equal('type', ['manual']) + ], APP_LIMIT_COUNT); + $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ + Query::equal('type', ['vcs']) + ], APP_LIMIT_COUNT); + + /** Get VCS repos connected */ + $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ], APP_LIMIT_COUNT); + + /** Get Total Teams */ + $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ], APP_LIMIT_COUNT); + } else { + $statsPerProject['custom_organization_members'] = 0; + } + + /** Get Email and Name of the project owner */ + if ($teamInternalId) { + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new \Exception('Membership not found. Skipping project : ' . $project->getId()); + } + + $userId = $membership->getAttribute('userId', null); + if ($userId) { + $user = $dbForConsole->getDocument('users', $userId); + $statsPerProject['email'] = $user->getAttribute('email', null); + $statsPerProject['name'] = $user->getAttribute('name', null); + } + } + + /** Get Domains */ + $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + /** Get Platforms */ + $platforms = $dbForConsole->find('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'web'; + })); + + $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'android'; + })); + + $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'apple'); + })); + + $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'flutter'); + })); + + $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; + + foreach ($flutterPlatforms as $flutterPlatform) { + $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { + return $platform['type'] === $flutterPlatform; + })); + } + + $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + /** Get Usage $statsPerProject */ + $periods = [ + 'infinity' => [ + 'period' => '1d', + 'limit' => 90, + ], + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + ]; + + Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { + foreach ($this->metrics as $key => $metric) { + foreach ($periods as $periodKey => $periodValue) { + $limit = $periodValue['limit']; + $period = $periodValue['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $statsPerProject[$key . '_' . $periodKey] = []; + foreach ($requestDocs as $requestDoc) { + $statsPerProject[$key . '_' . $periodKey][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); + // Calculate aggregate of each metric + $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); + } + } + }); + + if (isset($statsPerProject['email'])) { + /** Send data to mixpanel */ + $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ + 'name' => $statsPerProject['name'], + 'email' => $statsPerProject['email'] + ]); + + if (!$res) { + Console::error('Failed to create user profile for project: ' . $project->getId()); + } + } + + $event = new AnalyticsEvent(); + $event + ->setName('Project Daily Usage') + ->setProps($statsPerProject); + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + Console::error('Failed to create event for project: ' . $project->getId()); + } + } catch (\Exception $e) { + Console::error('Failed to send stats for project: ' . $project->getId()); + Console::error($e->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + /** + * @param Document $organization + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * @param Log $log + * @throws \Utopia\Database\Exception + */ + private function getStatsForOrganization(Document $organization, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void + { + try { + $statsPerOrganization = []; + + /** Organization name */ + $statsPerOrganization['name'] = $organization->getAttribute('name'); + + /** Get Email and of the organization owner */ + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$organization->getInternalId()]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId()); + } + + $userId = $membership->getAttribute('userId', null); + if ($userId) { + $user = $dbForConsole->getDocument('users', $userId); + $statsPerOrganization['email'] = $user->getAttribute('email', null); + } + + + /** Organization Creation Date */ + $statsPerOrganization['created'] = $organization->getAttribute('$createdAt'); + + /** Number of team members */ + $statsPerOrganization['members'] = $organization->getAttribute('total'); + + /** Number of projects in this organization */ + $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ + Query::equal('teamId', [$organization->getId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerOrganization['email'])) { + throw new \Exception('Email not found. Skipping organization : ' . $organization->getId()); + } + + $event = new AnalyticsEvent(); + $event + ->setName('Organization Daily Usage') + ->setProps($statsPerOrganization); + $res = $this->mixpanel->createEvent($event); + if (!$res) { + throw new \Exception('Failed to create event for organization : ' . $organization->getId()); + } + } catch (\Exception $e) { + Console::error($e->getMessage()); + } + } + + protected function getStatsPerUser(Document $user, Database $dbForConsole) + { + try { + $statsPerUser = []; + + /** Organization name */ + $statsPerUser['name'] = $user->getAttribute('name'); + + /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ + $statsPerUser['email'] = $user->getAttribute('email'); + + /** Organization Creation Date */ + $statsPerUser['created'] = $user->getAttribute('$createdAt'); + + /** Number of teams this user is a part of */ + $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ + Query::equal('userInternalId', [$user->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerUser['email'])) { + throw new \Exception('User has no email: ' . $user->getId()); + } + + /** Send data to mixpanel */ + $event = new AnalyticsEvent(); + $event + ->setName('User Daily Usage') + ->setProps($statsPerUser); + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + throw new \Exception('Failed to create user profile for user: ' . $user->getId()); + } + } catch (\Exception $e) { + Console::error($e->getMessage()); + } + } +} From 8bec64b2a2063f6384a578715ddb980226000532 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 30 Nov 2023 11:05:15 +0000 Subject: [PATCH 02/10] Update Hamster.php --- src/Appwrite/Platform/Tasks/Hamster.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index fea4c88ade..1df16fe9f8 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -78,9 +78,6 @@ class Hamster extends Action $next = new \DateTime($now->format("Y-m-d $jobInitTime")); $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); $delay = $next->getTimestamp() - $now->getTimestamp(); - - $delay = 5; - /** * If time passed for the target day. */ From d34050a5dff79bc37d38dd5c66776e66dda8b8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 16:22:26 +0000 Subject: [PATCH 03/10] chore: address review comments --- src/Appwrite/Event/Hamster.php | 16 ++-- src/Appwrite/Platform/Tasks/Hamster.php | 115 ++++++++---------------- 2 files changed, 48 insertions(+), 83 deletions(-) diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php index 9ae7303674..54cf0012fe 100644 --- a/src/Appwrite/Event/Hamster.php +++ b/src/Appwrite/Event/Hamster.php @@ -13,6 +13,10 @@ class Hamster extends Event protected ?Document $organization = null; protected ?Document $user = null; + const TYPE_PROJECT = 'project'; + const TYPE_ORGANISATION = 'organisation'; + const TYPE_USER = 'user'; + public function __construct(protected Connection $connection) { parent::__construct($connection); @@ -47,7 +51,7 @@ class Hamster extends Event /** * Sets the project for the hamster event. - * + * * @param Document $project */ public function setProject(Document $project): self @@ -59,7 +63,7 @@ class Hamster extends Event /** * Returns the set project for the hamster event. - * + * * @return Document */ public function getProject(): Document @@ -69,7 +73,7 @@ class Hamster extends Event /** * Sets the organization for the hamster event. - * + * * @param Document $organization */ public function setOrganization(Document $organization): self @@ -81,7 +85,7 @@ class Hamster extends Event /** * Returns the set organization for the hamster event. - * + * * @return string */ public function getOrganization(): Document @@ -91,7 +95,7 @@ class Hamster extends Event /** * Sets the user for the hamster event. - * + * * @param Document $user */ public function setUser(Document $user): self @@ -103,7 +107,7 @@ class Hamster extends Event /** * Returns the set user for the hamster event. - * + * * @return Document */ public function getUser(): Document diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index fea4c88ade..7a105ba450 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -3,31 +3,16 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Hamster as EventHamster; -use Appwrite\Network\Validator\Origin; use Exception; use Utopia\App; use Utopia\Platform\Action; -use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; -use Utopia\Analytics\Adapter\Mixpanel; -use Utopia\Analytics\Event; -use Utopia\Config\Config; use Utopia\Database\Document; -use Utopia\Pools\Group; class Hamster extends Action { - protected string $directory = '/usr/local'; - - protected string $path; - - protected string $date; - - protected Mixpanel $mixpanel; - public static function getName(): string { return 'hamster'; @@ -35,48 +20,30 @@ class Hamster extends Action public function __construct() { - $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', '')); - $this ->desc('Get stats for projects') - ->inject('pools') - ->inject('cache') + ->inject('queueForHamster') ->inject('dbForConsole') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) { - $this->action($pools, $cache, $dbForConsole); + ->callback(function (EventHamster $queueForHamster, Database $dbForConsole) { + $this->action($queueForHamster, $dbForConsole); }); } - private function getStatsPerProject(Group $pools, Database $dbForConsole) + public function action(EventHamster $queueForHamster, Database $dbForConsole): void { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools) { - $queue = $pools->get('queue')->pop(); - $connection = $queue->getResource(); - - $hamsterTask = new EventHamster($connection); - - $hamsterTask - ->setType('project') - ->setProject($project) - ->trigger(); - - $queue->reclaim(); - }); - } - - public function action(Group $pools, Cache $cache, Database $dbForConsole): void - { - Console::title('Cloud Hamster V1'); Console::success(APP_NAME . ' cloud hamster process has started'); $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes) + $now = new \DateTime(); $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); + $next = new \DateTime($now->format("Y-m-d $jobInitTime")); $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); + $delay = $next->getTimestamp() - $now->getTimestamp(); $delay = 5; @@ -91,29 +58,24 @@ class Hamster extends Action Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']'); - Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) { + Console::loop(function () use ($queueForHamster, $dbForConsole, $sleep) { $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds"); + Console::info("[{$now}] Queuing Cloud Usage Stats every {$sleep} seconds"); $loopStart = microtime(true); - /* Initialise new Utopia app */ - $app = new App('UTC'); - Console::info('Queuing stats for all projects'); - $this->getStatsPerProject($pools, $dbForConsole); + $this->getStatsPerProject($queueForHamster, $dbForConsole); Console::success('Completed queuing stats for all projects'); Console::info('Queuing stats for all organizations'); - $this->getStatsPerOrganization($pools, $dbForConsole); + $this->getStatsPerOrganization($queueForHamster, $dbForConsole); Console::success('Completed queuing stats for all organizations'); Console::info('Queuing stats for all users'); - $this->getStatsPerUser($pools, $dbForConsole); + $this->getStatsPerUser($queueForHamster, $dbForConsole); Console::success('Completed queuing stats for all users'); - $pools - ->get('console') - ->reclaim(); + $queue->reclaim(); $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); @@ -121,7 +83,7 @@ class Hamster extends Action }, $sleep, $delay); } - protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback) + protected function calculateByGroup(string $collection, Database $database, callable $callback) { $count = 0; $chunk = 0; @@ -134,7 +96,7 @@ class Hamster extends Action while ($sum === $limit) { $chunk++; - $results = $dbForConsole->find($collection, \array_merge([ + $results = $database->find($collection, \array_merge([ Query::limit($limit), Query::offset($count) ])); @@ -144,7 +106,7 @@ class Hamster extends Action Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents'); foreach ($results as $document) { - call_user_func($callback, $dbForConsole, $document); + call_user_func($callback, $database, $document); $count++; } } @@ -154,43 +116,42 @@ class Hamster extends Action Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } - protected function getStatsPerOrganization(Group $pools, Database $dbForConsole) + protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole) { - - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($pools) { + $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster) { try { - $queue = $pools->get('queue')->pop(); - $connection = $queue->getResource(); - - $hamsterTask = new EventHamster($connection); - - $hamsterTask - ->setType('organization') + $hamster + ->setType(EventHamster::TYPE_ORGANISATION) ->setOrganization($organization) ->trigger(); - - $queue->reclaim(); } catch (Exception $e) { Console::error($e->getMessage()); } }); } - protected function getStatsPerUser(Group $pools, Database $dbForConsole) + private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole) { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($pools) { + $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster) { try { - $queue = $pools->get('queue')->pop(); - $connection = $queue->getResource(); - - $hamsterTask = new EventHamster($connection); - - $hamsterTask - ->setType('user') + $hamster + ->setType(EventHamster::TYPE_PROJECT) + ->setProject($project) + ->trigger(); + } catch (Exception $e) { + Console::error($e->getMessage()); + } + }); + } + + protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole) + { + $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster) { + try { + $hamster + ->setType(EventHamster::TYPE_USER) ->setUser($user) ->trigger(); - - $queue->reclaim(); } catch (Exception $e) { Console::error($e->getMessage()); } From 4b7676158e540bc1711fc45981fc2fbc54d4466e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 16:24:11 +0000 Subject: [PATCH 04/10] chore: address review comments --- src/Appwrite/Platform/Tasks/Hamster.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 157f3db395..9ecab942ab 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -72,8 +72,6 @@ class Hamster extends Action $this->getStatsPerUser($queueForHamster, $dbForConsole); Console::success('Completed queuing stats for all users'); - $queue->reclaim(); - $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); From f3544485e54c44c63f228f53f6354016b374e364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 17:41:23 +0000 Subject: [PATCH 05/10] chore: address review comments --- app/init.php | 3 -- src/Appwrite/Platform/Tasks/Hamster.php | 21 ++++---- src/Appwrite/Platform/Workers/Hamster.php | 62 ++++++++--------------- 3 files changed, 33 insertions(+), 53 deletions(-) diff --git a/app/init.php b/app/init.php index c30eb77e85..2beeb28455 100644 --- a/app/init.php +++ b/app/init.php @@ -917,9 +917,6 @@ App::setResource('queueForCertificates', function (Connection $queue) { App::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); -App::setResource('queueForHamster', function (Connection $queue) { - return new Hamster($queue); -}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 9ecab942ab..1dca095e93 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -61,15 +61,15 @@ class Hamster extends Action $loopStart = microtime(true); Console::info('Queuing stats for all projects'); - $this->getStatsPerProject($queueForHamster, $dbForConsole); + $this->getStatsPerProject($queueForHamster, $dbForConsole, $loopStart); Console::success('Completed queuing stats for all projects'); Console::info('Queuing stats for all organizations'); - $this->getStatsPerOrganization($queueForHamster, $dbForConsole); + $this->getStatsPerOrganization($queueForHamster, $dbForConsole, $loopStart); Console::success('Completed queuing stats for all organizations'); Console::info('Queuing stats for all users'); - $this->getStatsPerUser($queueForHamster, $dbForConsole); + $this->getStatsPerUser($queueForHamster, $dbForConsole, $loopStart); Console::success('Completed queuing stats for all users'); $loopTook = microtime(true) - $loopStart; @@ -111,10 +111,11 @@ class Hamster extends Action Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } - protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole) + protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole, float $loopStart) { - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster) { + $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster, $loopStart) { try { + $organization->setAttribute('$time', $loopStart); $hamster ->setType(EventHamster::TYPE_ORGANISATION) ->setOrganization($organization) @@ -125,10 +126,11 @@ class Hamster extends Action }); } - private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole) + private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole, float $loopStart) { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster) { + $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster, $loopStart) { try { + $project->setAttribute('$time', $loopStart); $hamster ->setType(EventHamster::TYPE_PROJECT) ->setProject($project) @@ -139,10 +141,11 @@ class Hamster extends Action }); } - protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole) + protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole, float $loopStart) { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster) { + $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster, $loopStart) { try { + $user->setAttribute('$time', $loopStart); $hamster ->setType(EventHamster::TYPE_USER) ->setUser($user) diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php index d1c03744c8..2931741c84 100644 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -2,8 +2,6 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Usage\Stats; -use Appwrite\Event\Event; use Appwrite\Event\Hamster as EventHamster; use Appwrite\Network\Validator\Origin; use Utopia\Analytics\Adapter\Mixpanel; @@ -37,12 +35,6 @@ class Hamster extends Action 'usage_executions' => 'executions.$all.compute.total', ]; - protected string $directory = '/usr/local'; - - protected string $path; - - protected string $date; - protected Mixpanel $mixpanel; public static function getName(): string @@ -55,19 +47,13 @@ class Hamster extends Action */ public function __construct() { - $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', '')); - $this ->desc('Hamster worker') ->inject('message') ->inject('pools') ->inject('cache') ->inject('dbForConsole') - ->inject('queueForHamster') - ->inject('queueForEvents') - ->inject('usage') - ->inject('log') - ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $pools, $cache, $dbForConsole, $queueForHamster, $queueForEvents, $usage, $log)); + ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole) => $this->action($message, $pools, $cache, $dbForConsole)); } /** @@ -75,24 +61,17 @@ class Hamster extends Action * @param Group $pools * @param Cache $cache * @param Database $dbForConsole - * @param EventHamster $queueForHamster - * @param Event $queueForEvents - * @param Stats $usage - * @param Log $log + * * @return void - * @throws Authorization - * @throws Structure * @throws \Utopia\Database\Exception - * @throws Conflict */ - public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log): void + public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole): void { - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new \Exception('Missing payload'); + $token = App::getEnv('_APP_MIXPANEL_TOKEN', ''); + if (empty($token)) { + throw new \Exception('Missing MixPanel Token'); } - + $this->mixpanel = new Mixpanel($token); $payload = $message->getPayload() ?? []; @@ -103,13 +82,13 @@ class Hamster extends Action $type = $payload['type'] ?? ''; switch ($type) { - case 'project': - $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole, $log); + case EventHamster::TYPE_PROJECT: + $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole); break; - case 'organization': - $this->getStatsForOrganization(new Document($payload['organization']), $pools, $cache, $dbForConsole, $log); + case EventHamster::TYPE_ORGANISATION: + $this->getStatsForOrganization(new Document($payload['organization']), $dbForConsole); break; - case 'user': + case EventHamster::TYPE_USER: $this->getStatsPerUser(new Document($payload['user']), $dbForConsole); break; } @@ -120,10 +99,9 @@ class Hamster extends Action * @param Group $pools * @param Cache $cache * @param Database $dbForConsole - * @param Log $log * @throws \Utopia\Database\Exception */ - private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void + private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole): void { /** * Skip user projects with id 'console' @@ -149,7 +127,7 @@ class Hamster extends Action $statsPerProject = []; - $statsPerProject['time'] = microtime(true); + $statsPerProject['time'] = $project->getAttribute('$time'); /** Get Project ID */ $statsPerProject['project_id'] = $project->getId(); @@ -354,20 +332,20 @@ class Hamster extends Action /** * @param Document $organization - * @param Group $pools - * @param Cache $cache * @param Database $dbForConsole - * @param Log $log * @throws \Utopia\Database\Exception */ - private function getStatsForOrganization(Document $organization, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void + private function getStatsForOrganization(Document $organization, Database $dbForConsole): void { try { $statsPerOrganization = []; + $statsPerOrganization['time'] = $organization->getAttribute('$time'); + /** Organization name */ $statsPerOrganization['name'] = $organization->getAttribute('name'); + /** Get Email and of the organization owner */ $membership = $dbForConsole->findOne('memberships', [ Query::equal('teamInternalId', [$organization->getInternalId()]), @@ -383,7 +361,6 @@ class Hamster extends Action $statsPerOrganization['email'] = $user->getAttribute('email', null); } - /** Organization Creation Date */ $statsPerOrganization['created'] = $organization->getAttribute('$createdAt'); @@ -418,6 +395,8 @@ class Hamster extends Action try { $statsPerUser = []; + $statsPerUser['time'] = $user->getAttribute('$time'); + /** Organization name */ $statsPerUser['name'] = $user->getAttribute('name'); @@ -442,6 +421,7 @@ class Hamster extends Action $event ->setName('User Daily Usage') ->setProps($statsPerUser); + $res = $this->mixpanel->createEvent($event); if (!$res) { From 799fe2acca6871821256f8f340e596674def159f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 17:43:07 +0000 Subject: [PATCH 06/10] chore: address review comments --- app/init.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/init.php b/app/init.php index 2beeb28455..2c0219eec2 100644 --- a/app/init.php +++ b/app/init.php @@ -72,7 +72,6 @@ use Ahc\Jwt\JWTException; use Appwrite\Event\Build; use Appwrite\Event\Certificate; use Appwrite\Event\Func; -use Appwrite\Event\Hamster; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; From 0382b2250aec40974e810e3365903f126b6e38ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 19:22:35 +0000 Subject: [PATCH 07/10] chore: address review comments --- src/Appwrite/Platform/Workers/Hamster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php index 2931741c84..c8ab1952cc 100644 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -61,7 +61,7 @@ class Hamster extends Action * @param Group $pools * @param Cache $cache * @param Database $dbForConsole - * + * * @return void * @throws \Utopia\Database\Exception */ From d4ff6961734e16b4159f5a8cadf8a9dfdd1f0bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 20:35:18 +0000 Subject: [PATCH 08/10] chore: fix linter issues --- src/Appwrite/Event/Hamster.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php index 54cf0012fe..5d79fce568 100644 --- a/src/Appwrite/Event/Hamster.php +++ b/src/Appwrite/Event/Hamster.php @@ -13,9 +13,9 @@ class Hamster extends Event protected ?Document $organization = null; protected ?Document $user = null; - const TYPE_PROJECT = 'project'; - const TYPE_ORGANISATION = 'organisation'; - const TYPE_USER = 'user'; + public const TYPE_PROJECT = 'project'; + public const TYPE_ORGANISATION = 'organisation'; + public const TYPE_USER = 'user'; public function __construct(protected Connection $connection) { From 49bb3444bf1d7194871e0285e59427c430f6a253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 21:11:31 +0000 Subject: [PATCH 09/10] chore: add logs --- docker-compose.yml | 34 ++++++++++++++++++++--- src/Appwrite/Platform/Workers/Hamster.php | 7 +++-- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index da9b0e51e1..97cf7e5136 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -722,6 +722,34 @@ services: <<: *x-logging container_name: appwrite-worker-hamster image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_MIXPANEL_TOKEN + + appwrite-hamster-scheduler: + entrypoint: hamster + <<: *x-logging + container_name: appwrite-hamster-scheduler + image: appwrite-dev networks: - appwrite volumes: @@ -743,10 +771,8 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_USAGE_STATS - - _APP_LOGGING_CONFIG - - _APP_LOGGING_PROVIDER - - _APP_MIXPANEL_TOKEN + - _APP_HAMSTER_TIME + - _APP_HAMSTER_INTERVAL openruntimes-executor: container_name: openruntimes-executor diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php index c8ab1952cc..e911bb6c7a 100644 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -111,8 +111,7 @@ class Hamster extends Action return; } - Console::log("Getting stats for {$project->getId()}"); - + Console::log("Getting stats for Project {$project->getId()}"); try { $db = $project->getAttribute('database'); @@ -337,6 +336,8 @@ class Hamster extends Action */ private function getStatsForOrganization(Document $organization, Database $dbForConsole): void { + Console::log("Getting stats for Organization {$organization->getId()}"); + try { $statsPerOrganization = []; @@ -392,6 +393,8 @@ class Hamster extends Action protected function getStatsPerUser(Document $user, Database $dbForConsole) { + Console::log("Getting stats for User {$user->getId()}"); + try { $statsPerUser = []; From 8218e95f17b3c4449d45177928d0efa5b652a0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Thu, 30 Nov 2023 21:26:07 +0000 Subject: [PATCH 10/10] chore: revert console --- app/console | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/console b/app/console index 49d039ed07..ab9ef73fe0 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 49d039ed07628155e7f56e2c997fcef90ecde267 +Subproject commit ab9ef73fe0e74bdbb195331bb868af3ede7d1aa3