diff --git a/.env b/.env index 6cc19b399b..c1ae5ad046 100644 --- a/.env +++ b/.env @@ -45,6 +45,7 @@ _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= +_APP_USERS_STATS_RECIPIENTS= _APP_HAMSTER_INTERVAL=86400 _APP_HAMSTER_TIME=21:00 _APP_MIXPANEL_TOKEN= diff --git a/Dockerfile b/Dockerfile index bf3f4f1634..ec5cad2eab 100755 --- a/Dockerfile +++ b/Dockerfile @@ -119,6 +119,7 @@ RUN mkdir -p /storage/uploads && \ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ chmod +x /usr/local/bin/clear-card-cache && \ + chmod +x /usr/local/bin/calc-users-stats && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ diff --git a/bin/calc-users-stats b/bin/calc-users-stats new file mode 100644 index 0000000000..60472ed7a2 --- /dev/null +++ b/bin/calc-users-stats @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php calc-users-stats $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 1eec45940f..20d6b0ae39 100644 --- a/composer.json +++ b/composer.json @@ -73,7 +73,8 @@ "phpmailer/phpmailer": "6.6.0", "chillerlan/php-qrcode": "4.3.3", "adhocore/jwt": "1.1.2", - "slickdeals/statsd": "3.1.0" + "slickdeals/statsd": "3.1.0", + "league/csv": "9.7.1" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index 5349ca36b1..a6be48eacd 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ca2a083ff1c0d0c77942674400137793", + "content-hash": "146a894efdd38248478ef09e28f31978", "packages": [ { "name": "adhocore/jwt", @@ -408,21 +408,21 @@ }, { "name": "guzzlehttp/guzzle", - "version": "7.6.1", + "version": "7.7.0", "source": { "type": "git", "url": "https://github.com/guzzle/guzzle.git", - "reference": "8444a2bacf1960bc6a2b62ed86b8e72e11eebe51" + "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/8444a2bacf1960bc6a2b62ed86b8e72e11eebe51", - "reference": "8444a2bacf1960bc6a2b62ed86b8e72e11eebe51", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/fb7566caccf22d74d1ab270de3551f72a58399f5", + "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5", "shasum": "" }, "require": { "ext-json": "*", - "guzzlehttp/promises": "^1.5", + "guzzlehttp/promises": "^1.5.3 || ^2.0", "guzzlehttp/psr7": "^1.9.1 || ^2.4.5", "php": "^7.2.5 || ^8.0", "psr/http-client": "^1.0", @@ -434,7 +434,8 @@ "require-dev": { "bamarni/composer-bin-plugin": "^1.8.1", "ext-curl": "*", - "php-http/client-integration-tests": "^3.0", + "php-http/client-integration-tests": "dev-master#2c025848417c1135031fdf9c728ee53d0a7ceaee as 3.0.999", + "php-http/message-factory": "^1.1", "phpunit/phpunit": "^8.5.29 || ^9.5.23", "psr/log": "^1.1 || ^2.0 || ^3.0" }, @@ -513,7 +514,7 @@ ], "support": { "issues": "https://github.com/guzzle/guzzle/issues", - "source": "https://github.com/guzzle/guzzle/tree/7.6.1" + "source": "https://github.com/guzzle/guzzle/tree/7.7.0" }, "funding": [ { @@ -529,38 +530,37 @@ "type": "tidelift" } ], - "time": "2023-05-15T20:43:01+00:00" + "time": "2023-05-21T14:04:53+00:00" }, { "name": "guzzlehttp/promises", - "version": "1.5.2", + "version": "2.0.0", "source": { "type": "git", "url": "https://github.com/guzzle/promises.git", - "reference": "b94b2807d85443f9719887892882d0329d1e2598" + "reference": "3a494dc7dc1d7d12e511890177ae2d0e6c107da6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/promises/zipball/b94b2807d85443f9719887892882d0329d1e2598", - "reference": "b94b2807d85443f9719887892882d0329d1e2598", + "url": "https://api.github.com/repos/guzzle/promises/zipball/3a494dc7dc1d7d12e511890177ae2d0e6c107da6", + "reference": "3a494dc7dc1d7d12e511890177ae2d0e6c107da6", "shasum": "" }, "require": { - "php": ">=5.5" + "php": "^7.2.5 || ^8.0" }, "require-dev": { - "symfony/phpunit-bridge": "^4.4 || ^5.1" + "bamarni/composer-bin-plugin": "^1.8.1", + "phpunit/phpunit": "^8.5.29 || ^9.5.23" }, "type": "library", "extra": { - "branch-alias": { - "dev-master": "1.5-dev" + "bamarni-bin": { + "bin-links": true, + "forward-command": false } }, "autoload": { - "files": [ - "src/functions_include.php" - ], "psr-4": { "GuzzleHttp\\Promise\\": "src/" } @@ -597,7 +597,7 @@ ], "support": { "issues": "https://github.com/guzzle/promises/issues", - "source": "https://github.com/guzzle/promises/tree/1.5.2" + "source": "https://github.com/guzzle/promises/tree/2.0.0" }, "funding": [ { @@ -613,7 +613,7 @@ "type": "tidelift" } ], - "time": "2022-08-28T14:55:35+00:00" + "time": "2023-05-21T13:50:22+00:00" }, { "name": "guzzlehttp/psr7", @@ -864,6 +864,90 @@ }, "time": "2022-11-29T16:25:20+00:00" }, + { + "name": "league/csv", + "version": "9.7.1", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/csv.git", + "reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/0ec57e8264ec92565974ead0d1724cf1026e10c1", + "reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "php": "^7.3 || ^8.0" + }, + "require-dev": { + "ext-curl": "*", + "ext-dom": "*", + "friendsofphp/php-cs-fixer": "^2.16", + "phpstan/phpstan": "^0.12.0", + "phpstan/phpstan-phpunit": "^0.12.0", + "phpstan/phpstan-strict-rules": "^0.12.0", + "phpunit/phpunit": "^9.5" + }, + "suggest": { + "ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes", + "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "League\\Csv\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ignace Nyamagana Butera", + "email": "nyamsprod@gmail.com", + "homepage": "https://github.com/nyamsprod/", + "role": "Developer" + } + ], + "description": "CSV data manipulation made easy in PHP", + "homepage": "http://csv.thephpleague.com", + "keywords": [ + "convert", + "csv", + "export", + "filter", + "import", + "read", + "transform", + "write" + ], + "support": { + "docs": "https://csv.thephpleague.com", + "issues": "https://github.com/thephpleague/csv/issues", + "rss": "https://github.com/thephpleague/csv/releases.atom", + "source": "https://github.com/thephpleague/csv" + }, + "funding": [ + { + "url": "https://github.com/sponsors/nyamsprod", + "type": "github" + } + ], + "time": "2021-04-17T16:32:08+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -3165,16 +3249,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.4", + "version": "v4.15.5", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "6bb5176bc4af8bcb7d926f88718db9b96a2d4290" + "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6bb5176bc4af8bcb7d926f88718db9b96a2d4290", - "reference": "6bb5176bc4af8bcb7d926f88718db9b96a2d4290", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/11e2663a5bc9db5d714eedb4277ee300403b4a9e", + "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e", "shasum": "" }, "require": { @@ -3215,9 +3299,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.4" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.5" }, - "time": "2023-03-05T19:49:14+00:00" + "time": "2023-05-19T20:20:00+00:00" }, { "name": "phar-io/manifest", @@ -3568,22 +3652,23 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.20.4", + "version": "1.21.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "7d568c87a9df9c5f7e8b5f075fc469aa8cb0a4cd" + "reference": "6df62b08faef4f899772bc7c3bbabb93d2b7a21c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/7d568c87a9df9c5f7e8b5f075fc469aa8cb0a4cd", - "reference": "7d568c87a9df9c5f7e8b5f075fc469aa8cb0a4cd", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/6df62b08faef4f899772bc7c3bbabb93d2b7a21c", + "reference": "6df62b08faef4f899772bc7c3bbabb93d2b7a21c", "shasum": "" }, "require": { "php": "^7.2 || ^8.0" }, "require-dev": { + "nikic/php-parser": "^4.15", "php-parallel-lint/php-parallel-lint": "^1.2", "phpstan/extension-installer": "^1.0", "phpstan/phpstan": "^1.5", @@ -3607,9 +3692,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.20.4" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.21.0" }, - "time": "2023-05-02T09:19:37+00:00" + "time": "2023-05-17T13:13:44+00:00" }, { "name": "phpunit/php-code-coverage", @@ -5457,5 +5542,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 591b051223..e01eaaacef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -135,6 +135,7 @@ services: - _APP_SMTP_SECURE - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD + - _APP_USERS_STATS_RECIPIENTS - _APP_USAGE_STATS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index b82dca422f..1073dfb0f7 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; +use Appwrite\Platform\Tasks\CalcUsersStats; class Tasks extends Service { @@ -41,6 +42,7 @@ class Tasks extends Service ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) - ->addAction(Specs::getName(), new Specs()); + ->addAction(Specs::getName(), new Specs()) + ->addAction(CalcUsersStats::getName(), new CalcUsersStats()); } } diff --git a/src/Appwrite/Platform/Tasks/CalcUsersStats.php b/src/Appwrite/Platform/Tasks/CalcUsersStats.php new file mode 100644 index 0000000000..6310fe17b4 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/CalcUsersStats.php @@ -0,0 +1,176 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + //docker compose exec -t appwrite calc-users-stats + + Console::title('Cloud Users calculation V1'); + Console::success(APP_NAME . ' cloud Users calculation has started'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/users_stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Project Name */ + $stats['Project Name'] = $project->getAttribute('name'); + + + /** Get Team Name and Id */ + $teamId = $project->getAttribute('teamId', null); + $teamName = null; + if ($teamId) { + $team = $dbForConsole->getDocument('teams', $teamId); + $teamName = $team->getAttribute('name'); + } + + $stats['Team ID'] = $teamId; + $stats['Team name'] = $teamName; + + /** Get Total Users */ + $stats['users'] = $dbForProject->count('users', []); + + $csv->insertOne(array_values($stats)); + } catch (\Throwable $th) { + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + $pools + ->get('console') + ->reclaim(); + + /** @var PHPMailer $mail */ + $mail = $register->get('smtp'); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); + + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Cloud Report for {$this->date}"; + $mail->Body = "Please find the daily cloud report atttached"; + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } +}