Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-21 16:08:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #7005 from appwrite/refactor-usage-sn

Browse source 
Refactor usage
This commit is contained in:
Torsten Dittmann 2024-01-24 10:00:07 +01:00 committed by GitHub
commit 47ea083ea4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
83 changed files with 4192 additions and 4888 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.env
View file

@ -4,12 +4,13 @@ _APP_WORKER_PER_CORE=6
_APP_CONSOLE_WHITELIST_ROOT=disabled
_APP_CONSOLE_WHITELIST_EMAILS=
_APP_CONSOLE_WHITELIST_IPS=
_APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io
_APP_SYSTEM_EMAIL_NAME=Appwrite
_APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
_APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io
_APP_SYSTEM_RESPONSE_FORMAT=
_APP_OPTIONS_ABUSE=disabled
_APP_OPTIONS_ROUTER_PROTECTION=disbled
_APP_OPTIONS_ROUTER_PROTECTION=disabled
_APP_OPTIONS_FORCE_HTTPS=disabled
_APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled
_APP_OPENSSL_KEY_V1=your-secret-key
@ -50,10 +51,6 @@ _APP_STORAGE_WASABI_BUCKET=
_APP_STORAGE_ANTIVIRUS=disabled
_APP_STORAGE_ANTIVIRUS_HOST=clamav
_APP_STORAGE_ANTIVIRUS_PORT=3310
_APP_INFLUXDB_HOST=influxdb
_APP_INFLUXDB_PORT=8086
_APP_STATSD_HOST=telegraf
_APP_STATSD_PORT=8125
_APP_SMTP_HOST=maildev
_APP_SMTP_PORT=1025
_APP_SMTP_SECURE=
@ -79,7 +76,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_USAGE_AGGREGATION_INTERVAL=5
_APP_USAGE_AGGREGATION_INTERVAL=60000
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_USAGE_STATS=enabled

12
CHANGES.md
View file

@ -1,3 +1,14 @@
# Version 1.4.14
## Changes
- New usage metrics collection flow [#7005](https://github.com/appwrite/appwrite/pull/7005)
- Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
- Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
- Removed usage labels dependency.
- Dropped type attribute from stats collection.
- Usage metrics are processed via new usage worker.
- updated Metric names.
# Version 1.4.13
## Notable changes
@ -49,6 +60,7 @@
* Use getQueueSize() in the Health service's get X queue endpoints [#7073](https://github.com/appwrite/appwrite/pull/7073)
* Delete linked VCS repos and comments [#7066](https://github.com/appwrite/appwrite/pull/7066)
# Version 1.4.9
## Bug fixes

1
CONTRIBUTING.md
View file

@ -210,7 +210,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi
│ ├── Task
│ ├── Template
│ ├── URL
│ ├── Usage
│ └── Utopia
└── tests # End to end & unit tests
├── e2e

8
Dockerfile
View file

@ -74,7 +74,6 @@ RUN mkdir -p /storage/uploads && \
# Executables
RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/upgrade && \
chmod +x /usr/local/bin/migrate && \
@ -95,7 +94,9 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-migrations && \
chmod +x /usr/local/bin/worker-hamster
chmod +x /usr/local/bin/worker-hamster && \
chmod +x /usr/local/bin/worker-usage
# Cloud Executabless
RUN chmod +x /usr/local/bin/hamster && \
@ -107,7 +108,8 @@ RUN chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/get-migration-stats
chmod +x /usr/local/bin/get-migration-stats && \
chmod +x /usr/local/bin/create-inf-metric
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

24
app/cli.php
View file

@ -125,30 +125,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
};
}, ['pools', 'dbForConsole', 'cache']);
CLI::setResource('influxdb', function (Registry $register) {
$client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if (in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
return $database;
}, ['register']);
CLI::setResource('queue', function (Group $pools) {
return $pools->get('queue')->pop()->getResource();
}, ['pools']);

135
app/config/collections.php
View file

@ -18,6 +18,63 @@ $auth = Config::getParam('auth', []);
*/
$commonCollections = [
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'resource',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_accessedAt',
'type' => Database::INDEX_KEY,
'attributes' => ['accessedAt'],
'lengths' => [],
'orders' => [],
],
[
'$id' => '_key_resource',
'type' => Database::INDEX_KEY,
'attributes' => ['resource'],
'lengths' => [],
'orders' => [],
],
],
],
'users' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('users'),
@ -1270,10 +1327,10 @@ $commonCollections = [
]
],
'stats' => [
'stats_v2' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('stats'),
'name' => 'Stats',
'$id' => ID::custom('stats_v2'),
'name' => 'stats_v2',
'attributes' => [
[
'$id' => ID::custom('metric'),
@ -1302,7 +1359,7 @@ $commonCollections = [
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => false,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
@ -1330,17 +1387,6 @@ $commonCollections = [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('type'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 1,
'signed' => false,
'required' => true,
'default' => 0, // 0 -> count, 1 -> sum
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
@ -1359,7 +1405,7 @@ $commonCollections = [
],
[
'$id' => ID::custom('_key_metric_period_time'),
'type' => Database::INDEX_KEY,
'type' => Database::INDEX_UNIQUE,
'attributes' => ['metric', 'period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
@ -2883,63 +2929,6 @@ $projectCollections = array_merge([
],
],
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'resource',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_accessedAt',
'type' => Database::INDEX_KEY,
'attributes' => ['accessedAt'],
'lengths' => [],
'orders' => [],
],
[
'$id' => '_key_resource',
'type' => Database::INDEX_KEY,
'attributes' => ['resource'],
'lengths' => [],
'orders' => [],
],
],
],
'variables' => [
'$collection' => Database::METADATA,
'$id' => 'variables',

20
app/config/locale/templates/email-inner-base.tpl
View file

@ -1,15 +1,9 @@
<p>{{hello}}</p>
<br>
<p>{{hello}},</p>
<p>{{body}}</p>
<a href="{{redirect}}" target="_blank">{{redirect}}</a>
<p><a href="{{redirect}}" target="_blank">{{redirect}}</a></p>
<p>{{footer}}</p>
<br>
<p>{{thanks}}</p>
<p>{{signature}}</p>
<p style="margin-bottom: 32px">
{{thanks}},
<br/>
{{signature}}
</p>

6
app/config/platforms.php
View file

@ -185,7 +185,7 @@ return [
[
'key' => 'web',
'name' => 'Console',
'version' => '0.3.0',
'version' => '0.5.0',
'url' => 'https://github.com/appwrite/sdk-for-console',
'package' => '',
'enabled' => true,
@ -195,8 +195,8 @@ return [
'family' => APP_PLATFORM_CONSOLE,
'prism' => 'javascript',
'source' => \realpath(__DIR__ . '/../sdks/console-web'),
'gitUrl' => 'git@github.com:appwrite/sdk-for-console.git',
'gitBranch' => 'dev',
'gitUrl' => 'https://github.com/appwrite/sdk-for-console.git',
'gitBranch' => 'main',
'gitRepoName' => 'sdk-for-console',
'gitUserName' => 'appwrite',
],

2
app/config/specs/open-api3-1.4.x-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-1.4.x-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-1.4.x-server.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-server.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-1.4.x-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-1.4.x-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-1.4.x-server.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-server.json
View file

File diff suppressed because one or more lines are too long

21
app/config/variables.php
View file

@ -115,14 +115,6 @@ return [
'question' => '',
'filter' => ''
],
// [
// 'name' => '_APP_CONSOLE_WHITELIST_DOMAINS',
// 'description' => 'This option allows you to limit creation of users to Appwrite console for users sharing the same email domains. This option is very useful for team working with company emails domain.\n\nTo enable this option, pass a list of allowed email domains separated by a comma.',
// 'introduction' => '',
// 'default' => '',
// 'required' => false,
// 'question' => '',
// ],
[
'name' => '_APP_CONSOLE_WHITELIST_IPS',
'description' => "This last option allows you to limit creation of users in Appwrite console for users sharing the same set of IP addresses. This option is very useful for team working with a VPN service or a company IP.\n\nTo enable/activate this option, pass a list of allowed IP addresses separated by a comma.",
@ -132,6 +124,15 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_CONSOLE_HOSTNAMES',
'description' => 'This option allows you to add additional hostnames to your Appwrite console. This option is very useful for allowing access to the console project from additional domains. To enable it, pass a list of allowed hostnames separated by a comma.',
'introduction' => '1.5.0',
'default' => '',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_SYSTEM_EMAIL_NAME',
'description' => 'This is the sender name value that will appear on email messages sent to developers from the Appwrite console. The default value is: \'Appwrite\'. You can use url encoded strings for spaces and special chars.',
@ -337,7 +338,7 @@ return [
],
[
'category' => 'InfluxDB',
'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
'description' => 'Deprecated since 1.4.8.',
'variables' => [
[
'name' => '_APP_INFLUXDB_HOST',
@ -361,7 +362,7 @@ return [
],
[
'category' => 'StatsD',
'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
'description' => 'Deprecated since 1.4.8.',
'variables' => [
[
'name' => '_APP_STATSD_HOST',

126
app/controllers/api/account.php
View file

@ -45,6 +45,8 @@ use Utopia\Validator\WhiteList;
use Appwrite\Auth\Validator\PasswordHistory;
use Appwrite\Auth\Validator\PasswordDictionary;
use Appwrite\Auth\Validator\PersonalData;
use Appwrite\Event\Delete;
use Appwrite\Hooks\Hooks;
$oauthDefaultSuccess = '/auth/oauth2/success';
$oauthDefaultFailure = '/auth/oauth2/failure';
@ -58,7 +60,6 @@ App::post('/v1/account')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'create')
@ -77,7 +78,8 @@ App::post('/v1/account')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$email = \strtolower($email);
if ('console' === $project->getId()) {
@ -118,6 +120,8 @@ App::post('/v1/account')
}
}
$hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
$passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
$password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
try {
@ -174,8 +178,6 @@ App::post('/v1/account/sessions/email')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:email'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createEmailSession')
@ -195,7 +197,8 @@ App::post('/v1/account/sessions/email')
->inject('locale')
->inject('geodb')
->inject('queueForEvents')
->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) {
->inject('hooks')
->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Hooks $hooks) {
$email = \strtolower($email);
$protocol = $request->getProtocol();
@ -214,6 +217,8 @@ App::post('/v1/account/sessions/email')
$user->setAttributes($profile->getArrayCopy());
$hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$detector = new Detector($request->getUserAgent('UNKNOWN'));
@ -428,8 +433,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
->label('abuse-limit', 50)
->label('abuse-key', 'ip:{ip}')
->label('docs', false)
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:{request.provider}'])
->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
->param('state', '', new Text(2048), 'OAuth2 state params.', true)
@ -802,7 +805,6 @@ App::get('/v1/account/identities')
->desc('List Identities')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listIdentities')
@ -857,7 +859,6 @@ App::delete('/v1/account/identities/:identityId')
->label('audits.event', 'identity.delete')
->label('audits.resource', 'identity/{request.$identityId}')
->label('audits.userId', '{user.$id}')
->label('usage.metric', 'identities.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteIdentity')
@ -1015,7 +1016,7 @@ App::post('/v1/account/sessions/magic-url')
$message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
$message
->setParam('{{body}}', $body)
->setParam('{{body}}', $body, escapeHtml: false)
->setParam('{{hello}}', $locale->getText("emails.magicSession.hello"))
->setParam('{{footer}}', $locale->getText("emails.magicSession.footer"))
->setParam('{{thanks}}', $locale->getText("emails.magicSession.thanks"))
@ -1070,11 +1071,11 @@ App::post('/v1/account/sessions/magic-url')
$emailVariables = [
'direction' => $locale->getText('settings.direction'),
/* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
/* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
'user' => '',
'team' => '',
'project' => $project->getAttribute('name'),
'redirect' => $url
'redirect' => $url,
'project' => $project->getAttribute('name')
];
$queueForMails
@ -1108,8 +1109,6 @@ App::put('/v1/account/sessions/magic-url')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:magic-url'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateMagicURLSession')
@ -1483,8 +1482,6 @@ App::post('/v1/account/sessions/anonymous')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:anonymous'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createAnonymousSession')
@ -1662,7 +1659,6 @@ App::get('/v1/account')
->desc('Get account')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'get')
@ -1683,7 +1679,6 @@ App::get('/v1/account/prefs')
->desc('Get account preferences')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getPrefs')
@ -1706,7 +1701,6 @@ App::get('/v1/account/sessions')
->desc('List sessions')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listSessions')
@ -1745,7 +1739,6 @@ App::get('/v1/account/logs')
->desc('List logs')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listLogs')
@ -1806,7 +1799,6 @@ App::get('/v1/account/sessions/:sessionId')
->desc('Get session')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getSession')
@ -1854,7 +1846,6 @@ App::patch('/v1/account/name')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateName')
@ -1889,7 +1880,6 @@ App::patch('/v1/account/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePassword')
@ -1907,7 +1897,8 @@ App::patch('/v1/account/password')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
->inject('hooks')
->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
// Check old password only if its an existing user.
if (!empty($user->getAttribute('passwordUpdate')) && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions'))) { // Double check user password
@ -1934,6 +1925,8 @@ App::patch('/v1/account/password')
}
}
$hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
$user
->setAttribute('password', $newPassword)
->setAttribute('passwordHistory', $history)
@ -1955,7 +1948,6 @@ App::patch('/v1/account/email')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateEmail')
@ -1972,7 +1964,9 @@ App::patch('/v1/account/email')
->inject('user')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) {
->inject('project')
->inject('hooks')
->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) {
// passwordUpdate will be empty if the user has never set a password
$passwordUpdate = $user->getAttribute('passwordUpdate');
@ -1983,6 +1977,8 @@ App::patch('/v1/account/email')
throw new Exception(Exception::USER_INVALID_CREDENTIALS);
}
$hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
$email = \strtolower($email);
// Makes sure this email is not already used in another identity
@ -2025,7 +2021,6 @@ App::patch('/v1/account/phone')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhone')
@ -2042,7 +2037,9 @@ App::patch('/v1/account/phone')
->inject('user')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) {
->inject('project')
->inject('hooks')
->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) {
// passwordUpdate will be empty if the user has never set a password
$passwordUpdate = $user->getAttribute('passwordUpdate');
@ -2053,6 +2050,8 @@ App::patch('/v1/account/phone')
throw new Exception(Exception::USER_INVALID_CREDENTIALS);
}
$hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
$user
->setAttribute('phone', $phone)
->setAttribute('phoneVerification', false) // After this user needs to confirm phone number again
@ -2084,7 +2083,6 @@ App::patch('/v1/account/prefs')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePrefs')
@ -2118,7 +2116,6 @@ App::patch('/v1/account/status')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateStatus')
@ -2162,7 +2159,6 @@ App::delete('/v1/account/sessions/:sessionId')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSession')
@ -2239,7 +2235,6 @@ App::patch('/v1/account/sessions/:sessionId')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateSession')
@ -2324,7 +2319,6 @@ App::delete('/v1/account/sessions')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSessions')
@ -2386,7 +2380,6 @@ App::post('/v1/account/recovery')
->label('audits.event', 'recovery.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createRecovery')
@ -2468,7 +2461,7 @@ App::post('/v1/account/recovery')
$message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
$message
->setParam('{{body}}', $body)
->setParam('{{body}}', $body, escapeHtml: false)
->setParam('{{hello}}', $locale->getText("emails.recovery.hello"))
->setParam('{{footer}}', $locale->getText("emails.recovery.footer"))
->setParam('{{thanks}}', $locale->getText("emails.recovery.thanks"))
@ -2523,11 +2516,11 @@ App::post('/v1/account/recovery')
$emailVariables = [
'direction' => $locale->getText('settings.direction'),
/* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
/* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
'user' => $profile->getAttribute('name'),
'team' => '',
'project' => $projectName,
'redirect' => $url
'redirect' => $url,
'project' => $projectName
];
$queueForMails
@ -2564,7 +2557,6 @@ App::put('/v1/account/recovery')
->label('audits.event', 'recovery.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateRecovery')
@ -2583,7 +2575,8 @@ App::put('/v1/account/recovery')
->inject('dbForProject')
->inject('project')
->inject('queueForEvents')
->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents) {
->inject('hooks')
->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Hooks $hooks) {
if ($password !== $passwordAgain) {
throw new Exception(Exception::USER_PASSWORD_MISMATCH);
}
@ -2617,6 +2610,8 @@ App::put('/v1/account/recovery')
$history = array_slice($history, (count($history) - $historyLimit), $historyLimit);
}
$hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
$profile = $dbForProject->updateDocument('users', $profile->getId(), $profile
->setAttribute('password', $newPassword)
->setAttribute('passwordHistory', $history)
@ -2651,7 +2646,6 @@ App::post('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createVerification')
@ -2719,7 +2713,7 @@ App::post('/v1/account/verification')
$message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
$message
->setParam('{{body}}', $body)
->setParam('{{body}}', $body, escapeHtml: false)
->setParam('{{hello}}', $locale->getText("emails.verification.hello"))
->setParam('{{footer}}', $locale->getText("emails.verification.footer"))
->setParam('{{thanks}}', $locale->getText("emails.verification.thanks"))
@ -2774,11 +2768,11 @@ App::post('/v1/account/verification')
$emailVariables = [
'direction' => $locale->getText('settings.direction'),
/* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
/* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
'user' => $user->getAttribute('name'),
'team' => '',
'project' => $projectName,
'redirect' => $url
'redirect' => $url,
'project' => $projectName
];
$queueForMails
@ -2812,7 +2806,6 @@ App::put('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateVerification')
@ -2873,7 +2866,6 @@ App::post('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createPhoneVerification')
@ -2973,7 +2965,6 @@ App::put('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneVerification')
@ -3024,3 +3015,40 @@ App::put('/v1/account/verification/phone')
$response->dynamic($verificationDocument, Response::MODEL_TOKEN);
});
App::delete('/v1/account')
->desc('Delete account')
->groups(['api', 'account'])
->label('event', 'users.[userId].delete')
->label('scope', 'account')
->label('audits.event', 'user.delete')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'account')
->label('sdk.method', 'delete')
->label('sdk.description', '/docs/references/account/delete.md')
->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
->label('sdk.response.model', Response::MODEL_NONE)
->inject('user')
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->inject('queueForDeletes')
->action(function (Document $user, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
if ($user->isEmpty()) {
throw new Exception(Exception::USER_NOT_FOUND);
}
$dbForProject->deleteDocument('users', $user->getId());
$queueForDeletes
->setType(DELETE_TYPE_DOCUMENT)
->setDocument($user);
$queueForEvents
->setParam('userId', $user->getId())
->setPayload($response->output($user, Response::MODEL_USER));
$response->noContent();
});

515
app/controllers/api/databases.php
View file

@ -402,7 +402,6 @@ App::post('/v1/databases')
->label('scope', 'databases.write')
->label('audits.event', 'database.create')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'create')
@ -476,7 +475,6 @@ App::get('/v1/databases')
->desc('List databases')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'list')
@ -524,7 +522,6 @@ App::get('/v1/databases/:databaseId')
->desc('Get database')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'get')
@ -639,7 +636,6 @@ App::put('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].update')
->label('audits.event', 'database.update')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'update')
@ -684,7 +680,6 @@ App::delete('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].delete')
->label('audits.event', 'database.delete')
->label('audits.resource', 'database/{request.databaseId}')
->label('usage.metric', 'databases.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'delete')
@ -729,8 +724,6 @@ App::post('/v1/databases/:databaseId/collections')
->label('scope', 'collections.write')
->label('audits.event', 'collection.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}')
->label('usage.metric', 'collections.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createCollection')
@ -796,8 +789,6 @@ App::get('/v1/databases/:databaseId/collections')
->desc('List collections')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollections')
@ -855,8 +846,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId')
->desc('Get collection')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getCollection')
@ -891,8 +880,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs')
->desc('List collection logs')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollectionLogs')
@ -990,8 +977,6 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].update')
->label('audits.event', 'collection.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateCollection')
@ -1060,8 +1045,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].delete')
->label('audits.event', 'collection.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteCollection')
@ -1117,8 +1100,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createStringAttribute')
@ -1175,8 +1156,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEmailAttribute')
@ -1219,8 +1198,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEnumAttribute')
@ -1268,8 +1245,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIpAttribute')
@ -1312,8 +1287,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createUrlAttribute')
@ -1356,8 +1329,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIntegerAttribute')
@ -1429,8 +1400,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createFloatAttribute')
@ -1505,8 +1474,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createBooleanAttribute')
@ -1548,8 +1515,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createDatetimeAttribute')
@ -1594,8 +1559,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createRelationshipAttribute')
@ -1673,8 +1636,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes')
->desc('List attributes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listAttributes')
@ -1748,8 +1709,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key')
->desc('Get attribute')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getAttribute')
@ -1827,8 +1786,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateStringAttribute')
@ -1868,8 +1825,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/email
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateEmailAttribute')
@ -1909,8 +1864,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateEnumAttribute')
@ -1952,8 +1905,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/ip/:k
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateIpAttribute')
@ -1993,8 +1944,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/url/:
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateUrlAttribute')
@ -2034,8 +1983,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateIntegerAttribute')
@ -2085,8 +2032,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateFloatAttribute')
@ -2136,8 +2081,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/boole
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateBooleanAttribute')
@ -2176,8 +2119,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/datet
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'documents.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateDatetimeAttribute')
@ -2216,8 +2157,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'documents.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateRelationshipAttribute')
@ -2272,8 +2211,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteAttribute')
@ -2383,8 +2320,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
->label('scope', 'collections.write')
->label('audits.event', 'index.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createIndex')
@ -2543,8 +2478,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes')
->desc('List indexes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listIndexes')
@ -2608,8 +2541,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->desc('Get index')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getIndex')
@ -2652,8 +2583,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update')
->label('audits.event', 'index.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteIndex')
@ -2718,8 +2647,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
->label('scope', 'documents.write')
->label('audits.event', 'document.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'documents.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -2956,8 +2883,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
->desc('List documents')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocuments')
@ -3083,8 +3008,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('Get document')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getDocument')
@ -3178,8 +3101,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('List document logs')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocumentLogs')
@ -3282,8 +3203,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
->label('scope', 'documents.write')
->label('audits.event', 'document.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
->label('usage.metric', 'documents.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -3512,8 +3431,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete')
->label('audits.event', 'document.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
->label('usage.metric', 'documents.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -3630,112 +3547,72 @@ App::get('/v1/databases/usage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_DATABASES)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_DATABASES,
METRIC_COLLECTIONS,
METRIC_DOCUMENTS,
];
$metrics = [
'databases.$all.count.total',
'documents.$all.count.total',
'collections.$all.count.total',
'databases.$all.requests.create',
'databases.$all.requests.read',
'databases.$all.requests.update',
'databases.$all.requests.delete',
'collections.$all.requests.create',
'collections.$all.requests.read',
'collections.$all.requests.update',
'collections.$all.requests.delete',
'documents.$all.requests.create',
'documents.$all.requests.read',
'documents.$all.requests.update',
'documents.$all.requests.delete'
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// Added 3'rd level to Index [period, metric, time] because of order by.
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'databasesCount' => $stats['databases.$all.count.total'] ?? [],
'documentsCount' => $stats['documents.$all.count.total'] ?? [],
'collectionsCount' => $stats['collections.$all.count.total'] ?? [],
'documentsCreate' => $stats['documents.$all.requests.create'] ?? [],
'documentsRead' => $stats['documents.$all.requests.read'] ?? [],
'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [],
'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [],
'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [],
'collectionsRead' => $stats['collections.$all.requests.read'] ?? [],
'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [],
'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [],
'databasesCreate' => $stats['databases.$all.requests.create'] ?? [],
'databasesRead' => $stats['databases.$all.requests.read'] ?? [],
'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [],
'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASES);
}
$response->dynamic(new Document([
'range' => $range,
'databasesTotal' => $usage[$metrics[0]]['total'],
'collectionsTotal' => $usage[$metrics[1]]['total'],
'documentsTotal' => $usage[$metrics[2]]['total'],
'databases' => $usage[$metrics[0]]['data'],
'collections' => $usage[$metrics[1]]['data'],
'documents' => $usage[$metrics[2]]['data'],
]), Response::MODEL_USAGE_DATABASES);
});
App::get('/v1/databases/:databaseId/usage')
@ -3749,102 +3626,76 @@ App::get('/v1/databases/:databaseId/usage')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_DATABASE)
->param('databaseId', '', new UID(), 'Database ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$database = $dbForProject->getDocument('databases', $databaseId);
$metrics = [
'collections.' . $databaseId . '.count.total',
'collections.' . $databaseId . '.requests.create',
'collections.' . $databaseId . '.requests.read',
'collections.' . $databaseId . '.requests.update',
'collections.' . $databaseId . '.requests.delete',
'documents.' . $databaseId . '.count.total',
'documents.' . $databaseId . '.requests.create',
'documents.' . $databaseId . '.requests.read',
'documents.' . $databaseId . '.requests.update',
'documents.' . $databaseId . '.requests.delete'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// TODO@kodumbeats explore performance if query is ordered by time ASC
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [],
'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [],
'collectionsRead' => $stats["collections.{$databaseId}.requests.read"] ?? [],
'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [],
'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [],
'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [],
]);
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'collectionsTotal' => $usage[$metrics[0]]['total'],
'documentsTotal' => $usage[$metrics[1]]['total'],
'collections' => $usage[$metrics[0]]['data'],
'documents' => $usage[$metrics[1]]['data'],
]), Response::MODEL_USAGE_DATABASE);
});
App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
@ -3859,7 +3710,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION)
->param('databaseId', '', new UID(), 'Database ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
->param('collectionId', '', new UID(), 'Collection ID.')
->inject('response')
->inject('dbForProject')
@ -3873,84 +3724,60 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
];
$metrics = [
"documents.{$databaseId}/{$collectionId}.count.total",
"documents.{$databaseId}/{$collectionId}.requests.create",
"documents.{$databaseId}/{$collectionId}.requests.read",
"documents.{$databaseId}/{$collectionId}.requests.update",
"documents.{$databaseId}/{$collectionId}.requests.delete",
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
$response->dynamic(new Document([
'range' => $range,
'documentsTotal' => $usage[$metrics[0]]['total'],
'documents' => $usage[$metrics[0]]['data'],
]), Response::MODEL_USAGE_COLLECTION);
});

329
app/controllers/api/functions.php
View file

@ -6,13 +6,13 @@ use Appwrite\Event\Build;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\Validator\FunctionEvent;
use Appwrite\Utopia\Response\Model\Rule;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Database\Validator\CustomId;
use Appwrite\Messaging\Adapter\Realtime;
use Utopia\Validator\Assoc;
use Appwrite\Usage\Stats;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@ -456,9 +456,9 @@ App::get('/v1/functions/:functionId/usage')
->label('sdk.method', 'getFunctionUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTION)
->param('functionId', '', new UID(), 'Function ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $range, Response $response, Database $dbForProject) {
@ -469,97 +469,85 @@ App::get('/v1/functions/:functionId/usage')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
];
$metrics = [
"executions.$functionId.compute.total",
"executions.$functionId.compute.success",
"executions.$functionId.compute.failure",
"executions.$functionId.compute.time",
"builds.$functionId.compute.total",
"builds.$functionId.compute.success",
"builds.$functionId.compute.failure",
"builds.$functionId.compute.time",
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
'executionsSuccess' => $stats["executions.$functionId.compute.success"] ?? [],
'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
$response->dynamic(new Document([
'range' => $range,
'deploymentsTotal' => $usage[$metrics[0]]['total'],
'deploymentsStorageTotal' => $usage[$metrics[1]]['total'],
'buildsTotal' => $usage[$metrics[2]]['total'],
'buildsStorageTotal' => $usage[$metrics[3]]['total'],
'buildsTimeTotal' => $usage[$metrics[4]]['total'],
'executionsTotal' => $usage[$metrics[5]]['total'],
'executionsTimeTotal' => $usage[$metrics[6]]['total'],
'deployments' => $usage[$metrics[0]]['data'],
'deploymentsStorage' => $usage[$metrics[1]]['data'],
'builds' => $usage[$metrics[2]]['data'],
'buildsStorage' => $usage[$metrics[3]]['data'],
'buildsTime' => $usage[$metrics[4]]['data'],
'executions' => $usage[$metrics[5]]['data'],
'executionsTime' => $usage[$metrics[6]]['data'],
]), Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
->desc('Get functions usage')
->groups(['api', 'functions', 'usage'])
->groups(['api', 'functions'])
->label('scope', 'functions.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'functions')
@ -567,97 +555,87 @@ App::get('/v1/functions/usage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_FUNCTIONS,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_EXECUTIONS,
METRIC_EXECUTIONS_COMPUTE,
];
$metrics = [
'executions.$all.compute.total',
'executions.$all.compute.failure',
'executions.$all.compute.success',
'executions.$all.compute.time',
'builds.$all.compute.total',
'builds.$all.compute.failure',
'builds.$all.compute.success',
'builds.$all.compute.time',
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats[$metrics[0]] ?? [],
'executionsFailure' => $stats[$metrics[1]] ?? [],
'executionsSuccess' => $stats[$metrics[2]] ?? [],
'executionsTime' => $stats[$metrics[3]] ?? [],
'buildsTotal' => $stats[$metrics[4]] ?? [],
'buildsFailure' => $stats[$metrics[5]] ?? [],
'buildsSuccess' => $stats[$metrics[6]] ?? [],
'buildsTime' => $stats[$metrics[7]] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
}
$response->dynamic(new Document([
'range' => $range,
'functionsTotal' => $usage[$metrics[0]]['total'],
'deploymentsTotal' => $usage[$metrics[1]]['total'],
'deploymentsStorageTotal' => $usage[$metrics[2]]['total'],
'buildsTotal' => $usage[$metrics[3]]['total'],
'buildsStorageTotal' => $usage[$metrics[4]]['total'],
'buildsTimeTotal' => $usage[$metrics[5]]['total'],
'executionsTotal' => $usage[$metrics[6]]['total'],
'executionsTimeTotal' => $usage[$metrics[7]]['total'],
'functions' => $usage[$metrics[0]]['data'],
'deployments' => $usage[$metrics[1]]['data'],
'deploymentsStorage' => $usage[$metrics[2]]['data'],
'builds' => $usage[$metrics[3]]['data'],
'buildsStorage' => $usage[$metrics[4]]['data'],
'buildsTime' => $usage[$metrics[5]]['data'],
'executions' => $usage[$metrics[6]]['data'],
'executionsTime' => $usage[$metrics[7]]['data'],
]), Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')
@ -1520,11 +1498,11 @@ App::post('/v1/functions/:functionId/executions')
->inject('dbForProject')
->inject('user')
->inject('queueForEvents')
->inject('usage')
->inject('queueForUsage')
->inject('mode')
->inject('queueForFunctions')
->inject('geodb')
->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) {
->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode, Func $queueForFunctions, Reader $geodb) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
@ -1752,6 +1730,13 @@ App::post('/v1/functions/:functionId/executions')
->setAttribute('responseStatusCode', 500)
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
Console::error($th->getMessage());
} finally {
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
;
}
if ($function->getAttribute('logging')) {
@ -1759,14 +1744,6 @@ App::post('/v1/functions/:functionId/executions')
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
}
// TODO revise this later using route label
$usage
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);

204
app/controllers/api/project.php
View file

@ -3,8 +3,10 @@
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Datetime as DateTimeValidator;
use Utopia\Database\Exception\Duplicate as DuplicateException;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
@ -14,11 +16,10 @@ use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\Database\DateTime;
App::get('/v1/project/usage')
->desc('Get usage stats for a project')
->groups(['api'])
->groups(['api', 'usage'])
->label('scope', 'projects.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'project')
@ -26,100 +27,141 @@ App::get('/v1/project/usage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('startDate', '', new DateTimeValidator(), 'Starting date for the usage')
->param('endDate', '', new DateTimeValidator(), 'End date for the usage')
->param('period', '1d', new WhiteList(['1h', '1d']), 'Period used', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject) {
$stats = $total = $usage = [];
$format = 'Y-m-d 00:00:00';
$firstDay = (new DateTime($startDate))->format($format);
$lastDay = (new DateTime($endDate))->format($format);
$metrics = [
'project.$all.network.requests',
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$metrics = [
'total' => [
METRIC_EXECUTIONS,
METRIC_DOCUMENTS,
METRIC_DATABASES,
METRIC_USERS,
METRIC_BUCKETS,
METRIC_FILES_STORAGE
],
'period' => [
METRIC_NETWORK_REQUESTS,
METRIC_NETWORK_INBOUND,
METRIC_NETWORK_OUTBOUND,
METRIC_USERS,
METRIC_EXECUTIONS
]
];
$stats = [];
$factor = match ($period) {
'1h' => 3600,
'1d' => 86400,
};
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$limit = match ($period) {
'1h' => (new DateTime($startDate))->diff(new DateTime($endDate))->days * 24,
'1d' => (new DateTime($startDate))->diff(new DateTime($endDate))->days
};
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$format = match ($period) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
foreach ($metrics['total'] as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$total[$metric] = $result['value'] ?? 0;
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
foreach ($metrics['period'] as $metric) {
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::greaterThanEqual('time', $firstDay),
Query::lessThan('time', $lastDay),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'requests' => $stats[$metrics[0]] ?? [],
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
$now = time();
foreach ($metrics['period'] as $metric) {
$usage[$metric] = [];
$leap = $now - ($limit * $factor);
while ($leap < $now) {
$leap += $factor;
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
$executionsBreakdown = array_map(function ($function) use ($dbForProject) {
$id = $function->getId();
$name = $function->getAttribute('name');
$metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
$value = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
return [
'resourceId' => $id,
'name' => $name,
'value' => $value['value'] ?? 0,
];
}, $dbForProject->find('functions'));
$bucketsBreakdown = array_map(function ($bucket) use ($dbForProject) {
$id = $bucket->getId();
$name = $bucket->getAttribute('name');
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
$value = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
return [
'resourceId' => $id,
'name' => $name,
'value' => $value['value'] ?? 0,
];
}, $dbForProject->find('buckets'));
$response->dynamic(new Document([
'requests' => ($usage[METRIC_NETWORK_REQUESTS]),
'network' => ($usage[METRIC_NETWORK_INBOUND] + $usage[METRIC_NETWORK_OUTBOUND]),
'users' => ($usage[METRIC_USERS]),
'executions' => ($usage[METRIC_EXECUTIONS]),
'executionsTotal' => $total[METRIC_EXECUTIONS],
'documentsTotal' => $total[METRIC_DOCUMENTS],
'databasesTotal' => $total[METRIC_DATABASES],
'usersTotal' => $total[METRIC_USERS],
'bucketsTotal' => $total[METRIC_BUCKETS],
'filesStorageTotal' => $total[METRIC_FILES_STORAGE],
'executionsBreakdown' => $executionsBreakdown,
'bucketsBreakdown' => $bucketsBreakdown
]), Response::MODEL_USAGE_PROJECT);
});
// Variables
// Variables
App::post('/v1/project/variables')
->desc('Create Variable')
->groups(['api'])

116
app/controllers/api/projects.php
View file

@ -293,120 +293,6 @@ App::get('/v1/projects/:projectId')
$response->dynamic($project, Response::MODEL_PROJECT);
});
App::get('/v1/projects/:projectId/usage')
->desc('Get usage stats for a project')
->groups(['api', 'projects', 'usage'])
->label('scope', 'projects.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'projects')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
->param('projectId', '', new UID(), 'Project unique ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForConsole')
->inject('dbForProject')
->inject('register')
->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) {
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
throw new Exception(Exception::PROJECT_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$dbForProject->setNamespace("_{$project->getInternalId()}");
$metrics = [
'project.$all.network.requests',
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'requests' => $stats[$metrics[0]] ?? [],
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
});
App::patch('/v1/projects/:projectId')
->desc('Update project')
->groups(['api', 'projects'])
@ -1704,7 +1590,7 @@ App::get('/v1/projects/:projectId/templates/email/:type/:locale')
$message
->setParam('{{hello}}', $localeObj->getText("emails.{$type}.hello"))
->setParam('{{footer}}', $localeObj->getText("emails.{$type}.footer"))
->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'))
->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'), escapeHtml: false)
->setParam('{{thanks}}', $localeObj->getText("emails.{$type}.thanks"))
->setParam('{{signature}}', $localeObj->getText("emails.{$type}.signature"))
->setParam('{{direction}}', $localeObj->getText('settings.direction'));

307
app/controllers/api/storage.php
View file

@ -53,7 +53,6 @@ App::post('/v1/storage/buckets')
->label('event', 'buckets.[bucketId].create')
->label('audits.event', 'bucket.create')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'createBucket')
@ -149,7 +148,6 @@ App::get('/v1/storage/buckets')
->desc('List buckets')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listBuckets')
@ -198,7 +196,6 @@ App::get('/v1/storage/buckets/:bucketId')
->desc('Get bucket')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucket')
@ -227,7 +224,6 @@ App::put('/v1/storage/buckets/:bucketId')
->label('event', 'buckets.[bucketId].update')
->label('audits.event', 'bucket.update')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'updateBucket')
@ -295,7 +291,6 @@ App::delete('/v1/storage/buckets/:bucketId')
->label('audits.event', 'bucket.delete')
->label('event', 'buckets.[bucketId].delete')
->label('audits.resource', 'bucket/{request.bucketId}')
->label('usage.metric', 'buckets.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'deleteBucket')
@ -338,9 +333,7 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('audits.event', 'file.create')
->label('event', 'buckets.[bucketId].files.[fileId].create')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}')
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
@ -715,8 +708,6 @@ App::get('/v1/storage/buckets/:bucketId/files')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listFiles')
->label('sdk.description', '/docs/references/storage/list-files.md')
@ -796,8 +787,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFile')
->label('sdk.description', '/docs/references/storage/get-file.md')
@ -847,8 +836,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->label('cache', true)
->label('cache.resourceType', 'bucket/{request.bucketId}')
->label('cache.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFilePreview')
@ -1018,8 +1005,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->desc('Get file for download')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileDownload')
@ -1160,8 +1145,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->desc('Get file for view')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileView')
@ -1317,8 +1300,6 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].update')
->label('audits.event', 'file.update')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.update')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -1427,8 +1408,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].delete')
->label('audits.event', 'file.delete')
->label('audits.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.delete')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -1520,7 +1499,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage', 'usage'])
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
@ -1528,109 +1507,78 @@ App::get('/v1/storage/usage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_BUCKETS,
METRIC_FILES,
METRIC_FILES_STORAGE,
];
$metrics = [
'project.$all.storage.size',
'buckets.$all.count.total',
'buckets.$all.requests.create',
'buckets.$all.requests.read',
'buckets.$all.requests.update',
'buckets.$all.requests.delete',
'files.$all.storage.size',
'files.$all.count.total',
'files.$all.requests.create',
'files.$all.requests.read',
'files.$all.requests.update',
'files.$all.requests.delete',
];
$total = [];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'bucketsCount' => $stats['buckets.$all.count.total'],
'bucketsCreate' => $stats['buckets.$all.requests.create'],
'bucketsRead' => $stats['buckets.$all.requests.read'],
'bucketsUpdate' => $stats['buckets.$all.requests.update'],
'bucketsDelete' => $stats['buckets.$all.requests.delete'],
'storage' => $stats['project.$all.storage.size'],
'filesCount' => $stats['files.$all.count.total'],
'filesCreate' => $stats['files.$all.requests.create'],
'filesRead' => $stats['files.$all.requests.read'],
'filesUpdate' => $stats['files.$all.requests.update'],
'filesDelete' => $stats['files.$all.requests.delete'],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
}
$response->dynamic(new Document([
'range' => $range,
'bucketsTotal' => $usage[$metrics[0]]['total'],
'filesTotal' => $usage[$metrics[1]]['total'],
'filesStorageTotal' => $usage[$metrics[2]]['total'],
'buckets' => $usage[$metrics[0]]['data'],
'files' => $usage[$metrics[1]]['data'],
'storage' => $usage[$metrics[2]]['data'],
]), Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for a storage bucket')
->groups(['api', 'storage', 'usage'])
->desc('Get usage stats for storage bucket')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
@ -1639,7 +1587,7 @@ App::get('/v1/storage/:bucketId/usage')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS)
->param('bucketId', '', new UID(), 'Bucket ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) {
@ -1650,86 +1598,65 @@ App::get('/v1/storage/:bucketId/usage')
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
];
$metrics = [
"files.{$bucketId}.count.total",
"files.{$bucketId}.storage.size",
"files.{$bucketId}.requests.create",
"files.{$bucketId}.requests.read",
"files.{$bucketId}.requests.update",
"files.{$bucketId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'filesCount' => $stats[$metrics[0]],
'filesStorage' => $stats[$metrics[1]],
'filesCreate' => $stats[$metrics[2]],
'filesRead' => $stats[$metrics[3]],
'filesUpdate' => $stats[$metrics[4]],
'filesDelete' => $stats[$metrics[5]],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
$response->dynamic(new Document([
'range' => $range,
'filesTotal' => $usage[$metrics[0]]['total'],
'filesStorageTotal' => $usage[$metrics[1]]['total'],
'files' => $usage[$metrics[0]]['data'],
'storage' => $usage[$metrics[1]]['data'],
]), Response::MODEL_USAGE_BUCKETS);
});

8
app/controllers/api/teams.php
View file

@ -556,7 +556,7 @@ App::post('/v1/teams/:teamId/memberships')
$message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
$message
->setParam('{{body}}', $body)
->setParam('{{body}}', $body, escapeHtml: false)
->setParam('{{hello}}', $locale->getText("emails.invitation.hello"))
->setParam('{{footer}}', $locale->getText("emails.invitation.footer"))
->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks"))
@ -612,11 +612,11 @@ App::post('/v1/teams/:teamId/memberships')
$emailVariables = [
'owner' => $user->getAttribute('name'),
'direction' => $locale->getText('settings.direction'),
/* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
/* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
'user' => $user->getAttribute('name'),
'team' => $team->getAttribute('name'),
'project' => $projectName,
'redirect' => $url
'redirect' => $url,
'project' => $projectName
];
$queueForMails

229
app/controllers/api/users.php
View file

@ -39,10 +39,12 @@ use Utopia\Validator\Integer;
use Appwrite\Auth\Validator\PasswordHistory;
use Appwrite\Auth\Validator\PasswordDictionary;
use Appwrite\Auth\Validator\PersonalData;
use Appwrite\Hooks\Hooks;
/** TODO: Remove function when we move to using utopia/platform */
function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $queueForEvents): Document
function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks): Document
{
$plaintextPassword = $password;
$hashOptionsObject = (\is_string($hashOptions)) ? \json_decode($hashOptions, true) : $hashOptions; // Cast to JSON array
$passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
@ -65,13 +67,13 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) {
$personalDataValidator = new PersonalData($userId, $email, $name, $phone);
if (!$personalDataValidator->isValid($password)) {
if (!$personalDataValidator->isValid($plaintextPassword)) {
throw new Exception(Exception::USER_PASSWORD_PERSONAL_DATA);
}
}
$password = (!empty($password)) ? ($hash === 'plaintext' ? Auth::passwordHash($password, $hash, $hashOptionsObject) : $password) : null;
$user = $dbForProject->createDocument('users', new Document([
$user = new Document([
'$id' => $userId,
'$permissions' => [
Permission::read(Role::any()),
@ -97,7 +99,13 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
'tokens' => null,
'memberships' => null,
'search' => implode(' ', [$userId, $email, $phone, $name]),
]));
]);
if ($hash === 'plaintext') {
$hooks->trigger('passwordValidator', [$dbForProject, $project, $plaintextPassword, &$user, true]);
}
$user = $dbForProject->createDocument('users', $user);
} catch (Duplicate $th) {
throw new Exception(Exception::USER_ALREADY_EXISTS);
}
@ -114,7 +122,6 @@ App::post('/v1/users')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'create')
@ -131,8 +138,9 @@ App::post('/v1/users')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
$user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents);
->inject('hooks')
->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -146,7 +154,6 @@ App::post('/v1/users/bcrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createBcryptUser')
@ -162,8 +169,9 @@ App::post('/v1/users/bcrypt')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
$user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -177,7 +185,6 @@ App::post('/v1/users/md5')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createMD5User')
@ -193,8 +200,9 @@ App::post('/v1/users/md5')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
$user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -208,7 +216,6 @@ App::post('/v1/users/argon2')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createArgon2User')
@ -224,8 +231,9 @@ App::post('/v1/users/argon2')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
$user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -239,7 +247,6 @@ App::post('/v1/users/sha')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createSHAUser')
@ -256,14 +263,15 @@ App::post('/v1/users/sha')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$options = '{}';
if (!empty($passwordVersion)) {
$options = '{"version":"' . $passwordVersion . '"}';
}
$user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
$user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -277,7 +285,6 @@ App::post('/v1/users/phpass')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createPHPassUser')
@ -293,8 +300,9 @@ App::post('/v1/users/phpass')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
$user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -308,7 +316,6 @@ App::post('/v1/users/scrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptUser')
@ -329,7 +336,8 @@ App::post('/v1/users/scrypt')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$options = [
'salt' => $passwordSalt,
'costCpu' => $passwordCpu,
@ -338,7 +346,7 @@ App::post('/v1/users/scrypt')
'length' => $passwordLength
];
$user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
$user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -352,7 +360,6 @@ App::post('/v1/users/scrypt-modified')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptModifiedUser')
@ -371,8 +378,9 @@ App::post('/v1/users/scrypt-modified')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
$user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@ -383,7 +391,6 @@ App::get('/v1/users')
->desc('List users')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'list')
@ -432,7 +439,6 @@ App::get('/v1/users/:userId')
->desc('Get user')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'get')
@ -458,7 +464,6 @@ App::get('/v1/users/:userId/prefs')
->desc('Get user preferences')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'getPrefs')
@ -486,7 +491,6 @@ App::get('/v1/users/:userId/sessions')
->desc('List user sessions')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listSessions')
@ -528,7 +532,6 @@ App::get('/v1/users/:userId/memberships')
->desc('List user memberships')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listMemberships')
@ -568,7 +571,6 @@ App::get('/v1/users/:userId/logs')
->desc('List user logs')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listLogs')
@ -650,7 +652,6 @@ App::get('/v1/users/identities')
->desc('List Identities')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listIdentities')
@ -703,7 +704,6 @@ App::patch('/v1/users/:userId/status')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateStatus')
@ -739,7 +739,6 @@ App::put('/v1/users/:userId/labels')
->label('scope', 'users.write')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateLabels')
@ -777,7 +776,6 @@ App::patch('/v1/users/:userId/verification/phone')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhoneVerification')
@ -814,7 +812,6 @@ App::patch('/v1/users/:userId/name')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateName')
@ -852,7 +849,6 @@ App::patch('/v1/users/:userId/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePassword')
@ -866,7 +862,8 @@ App::patch('/v1/users/:userId/password')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
->inject('hooks')
->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$user = $dbForProject->getDocument('users', $userId);
@ -881,6 +878,8 @@ App::patch('/v1/users/:userId/password')
}
}
$hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
$newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
$historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
@ -917,7 +916,6 @@ App::patch('/v1/users/:userId/email')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmail')
@ -973,7 +971,6 @@ App::patch('/v1/users/:userId/phone')
->label('scope', 'users.write')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhone')
@ -1018,7 +1015,6 @@ App::patch('/v1/users/:userId/verification')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{request.userId}')
->label('audits.userId', '{request.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
@ -1051,7 +1047,6 @@ App::patch('/v1/users/:userId/prefs')
->groups(['api', 'users'])
->label('event', 'users.[userId].update.prefs')
->label('scope', 'users.write')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePrefs')
@ -1087,7 +1082,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSession')
@ -1131,7 +1125,6 @@ App::delete('/v1/users/:userId/sessions')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSessions')
@ -1174,7 +1167,6 @@ App::delete('/v1/users/:userId')
->label('scope', 'users.write')
->label('audits.event', 'user.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'delete')
@ -1217,7 +1209,6 @@ App::delete('/v1/users/identities/:identityId')
->label('scope', 'users.write')
->label('audits.event', 'identity.delete')
->label('audits.resource', 'identity/{request.$identityId}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteIdentity')
@ -1248,7 +1239,7 @@ App::delete('/v1/users/identities/:identityId')
App::get('/v1/users/usage')
->desc('Get usage stats for the users API')
->groups(['api', 'users', 'usage'])
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'users')
@ -1256,97 +1247,69 @@ App::get('/v1/users/usage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_USERS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->inject('register')
->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_USERS,
METRIC_SESSIONS,
];
$metrics = [
'users.$all.count.total',
'users.$all.requests.create',
'users.$all.requests.read',
'users.$all.requests.update',
'users.$all.requests.delete',
'sessions.$all.requests.create',
'sessions.$all.requests.delete',
"sessions.$provider.requests.create",
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $count => $metric) {
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'usersCount' => $stats['users.$all.count.total'] ?? [],
'usersCreate' => $stats['users.$all.requests.create'] ?? [],
'usersRead' => $stats['users.$all.requests.read'] ?? [],
'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_USERS);
$response->dynamic(new Document([
'range' => $range,
'usersTotal' => $usage[$metrics[0]]['total'],
'sessionsTotal' => $usage[$metrics[1]]['total'],
'users' => $usage[$metrics[0]]['data'],
'sessions' => $usage[$metrics[1]]['data'],
]), Response::MODEL_USAGE_USERS);
});

174
app/controllers/shared/api.php
View file

@ -8,8 +8,8 @@ use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Extend\Exception;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Request;
use Utopia\App;
@ -48,43 +48,95 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
return $label;
};
$databaseListener = function (string $event, Document $document, Stats $usage) {
$multiplier = 1;
$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
$value = 1;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$multiplier = -1;
$value = -1;
}
$collection = $document->getCollection();
switch ($collection) {
case 'users':
$usage->setParam('users.{scope}.count.total', 1 * $multiplier);
switch (true) {
case $document->getCollection() === 'teams':
$queueForUsage
->addMetric(METRIC_TEAMS, $value); // per project
break;
case 'databases':
$usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'users':
$queueForUsage
->addMetric(METRIC_USERS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case 'buckets':
$usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'sessions': // sessions
$queueForUsage
->addMetric(METRIC_SESSIONS, $value); //per project
break;
case 'deployments':
$usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
case $document->getCollection() === 'databases': // databases
$queueForUsage
->addMetric(METRIC_DATABASES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$queueForUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$collectionInternalId = $parts[3] ?? 0;
$queueForUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
->addMetric(METRIC_BUCKETS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$parts = explode('_', $document->getCollection());
$bucketInternalId = $parts[1];
$queueForUsage
->addMetric(METRIC_FILES, $value) // per project
->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
break;
case $document->getCollection() === 'functions':
$queueForUsage
->addMetric(METRIC_FUNCTIONS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'deployments':
$queueForUsage
->addMetric(METRIC_DEPLOYMENTS, $value) // per project
->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);
break;
default:
if (strpos($collection, 'bucket_') === 0) {
$usage
->setParam('bucketId', $document->getAttribute('bucketId'))
->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
->setParam('files.{scope}.count.total', 1 * $multiplier);
} elseif (strpos($collection, 'database_') === 0) {
$usage
->setParam('databaseId', $document->getAttribute('databaseId'));
if (strpos($collection, '_collection_') !== false) {
$usage
->setParam('collectionId', $document->getAttribute('$collectionId'))
->setParam('documents.{scope}.count.total', 1 * $multiplier);
} else {
$usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
}
}
break;
}
};
@ -103,8 +155,8 @@ App::init()
->inject('dbForProject')
->inject('mode')
->inject('queueForMails')
->inject('usage')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Stats $usage) use ($databaseListener) {
->inject('queueForUsage')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Usage $queueForUsage) use ($databaseListener) {
$route = $utopia->getRoute();
@ -189,19 +241,14 @@ App::init()
->setProject($project)
->setUser($user);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('project.{scope}.network.requests', 1)
->setParam('httpMethod', $request->getMethod())
->setParam('project.{scope}.network.inbound', 0)
->setParam('project.{scope}.network.outbound', 0);
$queueForDeletes->setProject($project);
$queueForDatabase->setProject($project);
$dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
;
$useCache = $route->getLabel('cache', false);
@ -365,14 +412,14 @@ App::shutdown()
->inject('user')
->inject('queueForEvents')
->inject('queueForAudits')
->inject('usage')
->inject('queueForUsage')
->inject('queueForDeletes')
->inject('queueForDatabase')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('mode')
->inject('dbForConsole')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Stats $usage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
$responsePayload = $response->getPayload();
@ -525,36 +572,25 @@ App::shutdown()
}
}
if (
App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
&& $project->getId()
&& !empty($route->getLabel('sdk.namespace', null))
) { // Don't calculate console usage on admin mode
$metric = $route->getLabel('usage.metric', '');
$usageParams = $route->getLabel('usage.params', []);
if (!empty($metric)) {
$usage->setParam($metric, 1);
foreach ($usageParams as $param) {
$param = $parseLabel($param, $responsePayload, $requestParams, $user);
$parts = explode(':', $param);
if (count($parts) != 2) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
}
$usage->setParam($parts[0], $parts[1]);
if ($project->getId() !== 'console') {
if ($mode !== APP_MODE_ADMIN) {
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
}
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$usage
->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
->setParam('project.{scope}.network.outbound', $response->getSize())
->submit();
$queueForUsage
->setProject($project)
->trigger();
}
/**

57
app/init.php
View file

@ -19,6 +19,7 @@ ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
use Appwrite\Event\Migration;
use Appwrite\Event\Usage;
use Appwrite\Extend\Exception;
use Appwrite\Auth\Auth;
use Appwrite\Event\Audit;
@ -32,7 +33,6 @@ use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Origin;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\URL\URL as AppwriteURL;
use Appwrite\Usage\Stats;
use Utopia\App;
use Utopia\Logger\Logger;
use Utopia\Cache\Adapter\Redis as RedisCache;
@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException;
use Appwrite\Event\Build;
use Appwrite\Event\Certificate;
use Appwrite\Event\Func;
use Appwrite\Hooks\Hooks;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
@ -80,6 +81,7 @@ use Utopia\Queue\Connection;
use Utopia\Storage\Storage;
use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub;
use Utopia\Validator\Range;
use Utopia\Validator\Hostname;
use Utopia\Validator\IP;
use Utopia\Validator\URL;
use Utopia\Validator\WhiteList;
@ -238,6 +240,7 @@ Config::load('platforms', __DIR__ . '/config/platforms.php');
Config::load('collections', __DIR__ . '/config/collections.php');
Config::load('runtimes', __DIR__ . '/config/runtimes.php');
Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php');
Config::load('usage', __DIR__ . '/config/usage.php');
Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes
Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes
Config::load('services', __DIR__ . '/config/services.php'); // List of services
@ -782,31 +785,7 @@ $register->set('db', function () {
return $pdo;
});
$register->set('influxdb', function () {
// Register DB connection
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if (empty($host) || empty($port)) {
return;
}
$driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
$client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
$client->setDriver($driver);
return $client;
});
$register->set('statsd', function () {
// Register DB connection
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
return $statsd;
});
$register->set('smtp', function () {
$mail = new PHPMailer(true);
@ -847,6 +826,9 @@ $register->set('passwordsDictionary', function () {
$register->set('promiseAdapter', function () {
return new Swoole();
});
$register->set('hooks', function () {
return new Hooks();
});
/*
* Localization
*/
@ -886,6 +868,10 @@ App::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
App::setResource('hooks', function ($register) {
return $register->get('hooks');
}, ['register']);
App::setResource('loggerBreadcrumbs', function () {
return [];
});
@ -925,15 +911,15 @@ App::setResource('queueForAudits', function (Connection $queue) {
App::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
App::setResource('queueForUsage', function (Connection $queue) {
return new Usage($queue);
}, ['queue']);
App::setResource('queueForCertificates', function (Connection $queue) {
return new Certificate($queue);
}, ['queue']);
App::setResource('queueForMigrations', function (Connection $queue) {
return new Migration($queue);
}, ['queue']);
App::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);
App::setResource('clients', function ($request, $console, $project) {
$console->setAttribute('platforms', [ // Always allow current host
'$collection' => ID::custom('platforms'),
@ -942,6 +928,21 @@ App::setResource('clients', function ($request, $console, $project) {
'hostname' => $request->getHostname(),
], Document::SET_TYPE_APPEND);
$hostnames = explode(',', App::getEnv('_APP_CONSOLE_HOSTNAMES', ''));
$validator = new Hostname();
foreach ($hostnames as $hostname) {
$hostname = trim($hostname);
if (!$validator->isValid($hostname)) {
continue;
}
$console->setAttribute('platforms', [
'$collection' => ID::custom('platforms'),
'type' => Origin::CLIENT_TYPE_WEB,
'name' => $hostname,
'hostname' => $hostname,
], Document::SET_TYPE_APPEND);
}
/**
* Get All verified client URLs for both console and current projects
* + Filter for duplicated entries

40
app/views/install/compose.phtml
View file

@ -71,7 +71,6 @@ services:
- mariadb
- redis
# - clamav
- influxdb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
@ -79,6 +78,7 @@ services:
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
- _APP_CONSOLE_WHITELIST_IPS
- _APP_CONSOLE_HOSTNAMES
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@ -106,8 +106,6 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@ -144,8 +142,6 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -273,7 +269,7 @@ services:
depends_on:
- redis
- mariadb
volumes:
volumes:
- appwrite-uploads:/storage/uploads:rw
- appwrite-cache:/storage/cache:rw
- appwrite-functions:/storage/functions:rw
@ -420,7 +416,7 @@ services:
depends_on:
- redis
- mariadb
volumes:
volumes:
- appwrite-config:/storage/config:rw
- appwrite-certificates:/storage/certificates:rw
environment:
@ -595,16 +591,15 @@ services:
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-usage:
appwrite-worker-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: usage
container_name: appwrite-usage
container_name: appwrite-worker-usage
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
@ -615,9 +610,6 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -751,27 +743,6 @@ services:
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
networks:
gateway:
name: gateway
@ -788,5 +759,4 @@ volumes:
appwrite-certificates:
appwrite-functions:
appwrite-builds:
appwrite-influxdb:
appwrite-config:

46
app/worker.php
View file

@ -11,11 +11,10 @@ use Appwrite\Event\Delete;
use Appwrite\Event\Func;
use Appwrite\Event\Hamster;
use Appwrite\Event\Mail;
use Appwrite\Event\Messaging;
use Appwrite\Event\Migration;
use Appwrite\Event\Phone;
use Appwrite\Event\Usage;
use Appwrite\Platform\Appwrite;
use Appwrite\Usage\Stats;
use Swoole\Runtime;
use Utopia\App;
use Utopia\Cache\Adapter\Sharding;
@ -23,6 +22,7 @@ use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Service;
@ -73,6 +73,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
return $adapter;
}, ['cache', 'register', 'message', 'dbForConsole']);
Server::setResource('project', function (Message $message, Database $dbForConsole) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
if ($project->getId() === 'console') {
return $project;
}
return $dbForConsole->getDocument('projects', $project->getId());
;
}, ['message', 'dbForConsole']);
Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
$databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
@ -104,6 +115,18 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
};
}, ['pools', 'dbForConsole', 'cache']);
Server::setResource('abuseRetention', function () {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
});
Server::setResource('auditRetention', function () {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
});
Server::setResource('executionRetention', function () {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
});
Server::setResource('cache', function (Registry $register) {
$pools = $register->get('pools');
$list = Config::getParam('pools-cache', []);
@ -120,9 +143,9 @@ Server::setResource('cache', function (Registry $register) {
return new Cache(new Sharding($adapters));
}, ['register']);
Server::setResource('log', fn() => new Log());
Server::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);
Server::setResource('queueForUsage', function (Connection $queue) {
return new Usage($queue);
}, ['queue']);
Server::setResource('queue', function (Group $pools) {
return $pools->get('queue')->pop()->getResource();
}, ['pools']);
@ -276,9 +299,12 @@ $worker
Console::error('[Error] Line: ' . $error->getLine());
});
$worker->workerStart()
->action(function () use ($workerName) {
Console::info("Worker $workerName started");
});
try {
$workerStart = $worker->getWorkerStart();
} catch (\Throwable $error) {
$worker->workerStart();
} finally {
Console::info("Worker $workerName started");
}
$worker->start();
$worker->start();

3
bin/create-inf-metric Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php create-inf-metric $@

3
bin/usage
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php usage $@

3
bin/worker-usage Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/worker.php usage $@

8
composer.json
View file

@ -46,13 +46,13 @@
"utopia-php/abuse": "0.33.*",
"utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.35.*",
"utopia-php/cache": "0.8.*",
"utopia-php/cache": "0.9.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.45.*",
"utopia-php/domains": "0.3.*",
"utopia-php/dsn": "0.1.*",
"utopia-php/framework": "0.31.1",
"utopia-php/framework": "0.33.*",
"utopia-php/image": "0.5.*",
"utopia-php/locale": "0.4.*",
"utopia-php/logger": "0.3.*",
@ -62,7 +62,7 @@
"utopia-php/platform": "0.5.*",
"utopia-php/pools": "0.4.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/queue": "0.5.*",
"utopia-php/queue": "0.6.*",
"utopia-php/registry": "0.5.*",
"utopia-php/storage": "0.18.*",
"utopia-php/swoole": "0.5.*",
@ -70,12 +70,10 @@
"utopia-php/websocket": "0.1.*",
"matomo/device-detector": "6.1.*",
"dragonmantank/cron-expression": "3.3.2",
"influxdb/influxdb-php": "1.15.2",
"phpmailer/phpmailer": "6.8.0",
"chillerlan/php-qrcode": "4.3.4",
"adhocore/jwt": "1.1.2",
"webonyx/graphql-php": "14.11.*",
"slickdeals/statsd": "3.1.0",
"league/csv": "9.7.1"
},
"repositories": [

866
composer.lock generated
View file

File diff suppressed because it is too large Load diff

77
docker-compose.yml
View file

@ -53,7 +53,7 @@ services:
DEBUG: false
TESTING: true
VERSION: dev
ports:
ports:
- 9501:80
networks:
- appwrite
@ -88,7 +88,7 @@ services:
- mariadb
- redis
# - clamav
entrypoint:
entrypoint:
- php
- -e
- app/http.php
@ -100,6 +100,7 @@ services:
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
- _APP_CONSOLE_WHITELIST_IPS
- _APP_CONSOLE_HOSTNAMES
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@ -127,8 +128,6 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@ -165,8 +164,6 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -195,7 +192,7 @@ services:
container_name: appwrite-realtime
image: appwrite-dev
restart: unless-stopped
ports:
ports:
- 9505:80
labels:
- "traefik.enable=true"
@ -303,7 +300,7 @@ services:
depends_on:
- redis
- mariadb
volumes:
volumes:
- appwrite-uploads:/storage/uploads:rw
- appwrite-cache:/storage/cache:rw
- appwrite-functions:/storage/functions:rw
@ -357,7 +354,7 @@ services:
image: appwrite-dev
networks:
- appwrite
volumes:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
@ -456,7 +453,7 @@ services:
depends_on:
- redis
- mariadb
volumes:
volumes:
- appwrite-config:/storage/config:rw
- appwrite-certificates:/storage/certificates:rw
- ./app:/usr/src/code/app
@ -648,19 +645,18 @@ services:
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-usage:
entrypoint: usage
appwrite-worker-usage:
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-usage
container_name: appwrite-worker-usage
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
depends_on:
- influxdb
- redis
- mariadb
environment:
- _APP_ENV
@ -671,9 +667,6 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -681,6 +674,7 @@ services:
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
appwrite-schedule:
entrypoint: schedule
@ -744,7 +738,7 @@ services:
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_MIXPANEL_TOKEN
appwrite-hamster-scheduler:
entrypoint: hamster
<<: *x-logging
@ -900,26 +894,6 @@ services:
# - appwrite
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
<<: *x-logging
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
# Dev Tools Start ------------------------------------------------------------------------------------------
#
# The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
@ -929,7 +903,6 @@ services:
# MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI.
# RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
# RedisCommander - A nice UI for exploring Redis data
# Chronograf - A nice UI for exploring InfluxDB data
# Webgrind - A nice UI for exploring and debugging code-level stuff
maildev: # used mainly for dev tests
@ -969,33 +942,13 @@ services:
# - REDIS_HOSTS=redis
# ports:
# - "8081:8081"
# chronograf:
# image: chronograf:1.6
# container_name: appwrite-chronograf
# restart: unless-stopped
# networks:
# - appwrite
# volumes:
# - appwrite-chronograf:/var/lib/chronograf
# ports:
# - "8888:8888"
# environment:
# - INFLUXDB_URL=http://influxdb:8086
# - KAPACITOR_URL=http://kapacitor:9092
# - AUTH_DURATION=48h
# - TOKEN_SECRET=duperduper5674829!jwt
# - GH_CLIENT_ID=d86f7145a41eacfc52cc
# - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
# - GH_ORGS=appwrite
# webgrind:
# image: 'jokkedk/webgrind:latest'
# volumes:
# - './debug:/tmp'
# ports:
# - '3001:80'
graphql-explorer:
container_name: appwrite-graphql-explorer
image: appwrite/altair:0.3.0
@ -1026,6 +979,4 @@ volumes:
appwrite-certificates:
appwrite-functions:
appwrite-builds:
appwrite-influxdb:
appwrite-config:
# appwrite-chronograf:

1
docs/references/account/delete.md Normal file
View file

@ -0,0 +1 @@
Delete the currently logged in user.

43
src/Appwrite/Event/Mail.php
View file

@ -13,6 +13,8 @@ class Mail extends Event
protected string $body = '';
protected array $smtp = [];
protected array $variables = [];
protected string $bodyTemplate = '';
protected array $attachment = [];
public function __construct(protected Connection $connection)
{
@ -115,6 +117,29 @@ class Mail extends Event
return $this->name;
}
/**
* Sets bodyTemplate for the mail event.
*
* @param string $bodyTemplate
* @return self
*/
public function setbodyTemplate(string $bodyTemplate): self
{
$this->bodyTemplate = $bodyTemplate;
return $this;
}
/**
* Returns subject for the mail event.
*
* @return string
*/
public function getbodyTemplate(): string
{
return $this->bodyTemplate;
}
/**
* Set SMTP Host
*
@ -313,6 +338,22 @@ class Mail extends Event
return $this;
}
public function setAttachment(string $content, string $filename, string $encoding = 'base64', string $type = 'plain/text')
{
$this->attachment = [
'content' => base64_encode($content),
'filename' => $filename,
'encoding' => $encoding,
'type' => $type,
];
return $this;
}
public function getAttachment(): array
{
return $this->attachment;
}
/**
* Executes the event and sends it to the mails worker.
*
@ -327,9 +368,11 @@ class Mail extends Event
'recipient' => $this->recipient,
'name' => $this->name,
'subject' => $this->subject,
'bodyTemplate' => $this->bodyTemplate,
'body' => $this->body,
'smtp' => $this->smtp,
'variables' => $this->variables,
'attachment' => $this->attachment,
'events' => Event::generateEvents($this->getEvent(), $this->getParams())
]);
}

26
src/Appwrite/Hooks/Hooks.php Normal file
View file

@ -0,0 +1,26 @@
<?php
namespace Appwrite\Hooks;
class Hooks
{
/**
* @var callable[] $hooks
*/
private static array $hooks = [];
public static function add(string $name, callable $action)
{
self::$hooks[$name] = $action;
}
/**
* @param mixed[] $params
*/
public function trigger(string $name, array $params = [])
{
if (isset(self::$hooks[$name])) {
call_user_func_array(self::$hooks[$name], $params);
}
}
}

3
src/Appwrite/Migration/Migration.php
View file

@ -77,7 +77,8 @@ abstract class Migration
'1.4.10' => 'V19',
'1.4.11' => 'V19',
'1.4.12' => 'V19',
'1.4.13' => 'V19'
'1.4.13' => 'V19',
'1.4.14' => 'V20'
];
/**

323
src/Appwrite/Migration/Version/V20.php Normal file
View file

@ -0,0 +1,323 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use PDOException;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Query;
class V20 extends Migration
{
/**
* @throws Throwable
*/
public function execute(): void
{
if ($this->project->getInternalId() == 'console') {
return;
}
/**
* Disable SubQueries for Performance.
*/
foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables'] as $name) {
Database::addFilter(
$name,
fn () => null,
fn () => []
);
}
$this->migrateUsageMetrics('project.$all.network.requests', 'network.requests');
$this->migrateUsageMetrics('project.$all.network.outbound', 'network.outbound');
$this->migrateUsageMetrics('project.$all.network.inbound', 'network.inbound');
$this->migrateUsageMetrics('users.$all.count.total', 'users');
$this->migrateSessionsMetric();
Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
Console::info('Migrating Functions');
$this->migrateFunctions();
Console::info('Migrating Databases');
$this->migrateDatabases();
Console::info('Migrating Collections');
$this->migrateCollections();
Console::info('Migrating Buckets');
$this->migrateBuckets();
}
/**
* @return void
* @throws Authorization
* @throws Exception
* @throws Structure
*/
protected function migrateSessionsMetric(): void
{
/**
* Creating inf metric
*/
Console::info('Migrating Sessions metric');
$sessionsCreated = $this->projectDB->sum('stats', 'value', [
Query::equal('metric', [
'sessions.email-password.requests.create',
'sessions.magic-url.requests.create',
'sessions.anonymous.requests.create',
'sessions.invites.requests.create',
'sessions.jwt.requests.create',
'sessions.phone.requests.create'
]),
Query::equal('period', ['1d']),
]);
$query = $this->projectDB->findOne('stats', [
Query::equal('metric', ['sessions.$all.requests.delete']),
Query::equal('period', ['1d']),
]);
$sessionsDeleted = $query['value'] ?? 0;
$value = $sessionsCreated - $sessionsDeleted;
$this->createInfMetric('sessions', $value);
}
/**
* @param string $metric
* @param int $value
* @return void
* @throws Exception
* @throws Authorization
* @throws Structure
*/
protected function createInfMetric(string $metric, int $value): void
{
try {
/**
* Creating inf metric
*/
console::log("Creating inf metric to {$metric}");
$id = \md5("_inf_{$metric}");
$this->projectDB->createDocument('stats', new Document([
'$id' => $id,
'metric' => $metric,
'period' => 'inf',
'value' => $value,
'time' => null,
'region' => 'default',
]));
} catch (Duplicate $th) {
console::log("Error while creating inf metric: duplicate id {$metric} {$id}");
}
}
/**
* @param string $from
* @param string $to
* @return void
* @throws Exception
*/
protected function migrateUsageMetrics(string $from, string $to): void
{
/**
* inf metric
*/
if (
str_contains($from, '$all') ||
str_contains($from, '.total')
) {
$query = $this->projectDB->sum('stats', 'value', [
Query::equal('metric', [$from]),
Query::equal('period', ['1d']),
]);
$value = $query ?? 0;
$this->createInfMetric($to, $value);
}
try {
/**
* Update old metric format to new
*/
$limit = 1000;
$sum = $limit;
$total = 0;
$latestDocument = null;
while ($sum === $limit) {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument !== null) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$stats = $this->projectDB->find('stats', \array_merge($paginationQueries, [
Query::equal('metric', [$from]),
]));
$sum = count($stats);
$total = $total + $sum;
foreach ($stats as $stat) {
$format = $stat['period'] === '1d' ? 'Y-m-d 00:00' : 'Y-m-d H:00';
$time = date($format, strtotime($stat['time']));
$this->projectDB->deleteDocument('stats', $stat->getId());
$stat->setAttribute('$id', \md5("{$time}_{$stat['period']}_{$to}"));
$stat->setAttribute('metric', $to);
$this->projectDB->createDocument('stats', $stat);
console::log("deleting metric {$from} and creating {$to}");
}
$latestDocument = !empty(array_key_last($stats)) ? $stats[array_key_last($stats)] : null;
}
} catch (Throwable $th) {
Console::warning("Error while updating metric {$from} " . $th->getMessage());
}
}
/**
* Migrate functions.
*
* @return void
* @throws \Exception
*/
private function migrateFunctions(): void
{
$this->migrateUsageMetrics('deployment.$all.storage.size', 'deployments.storage');
$this->migrateUsageMetrics('builds.$all.compute.total', 'builds');
$this->migrateUsageMetrics('builds.$all.compute.time', 'builds.compute');
$this->migrateUsageMetrics('executions.$all.compute.total', 'executions');
$this->migrateUsageMetrics('executions.$all.compute.time', 'executions.compute');
foreach ($this->documentsIterator('functions') as $function) {
Console::log("Migrating Functions usage stats of {$function->getId()} ({$function->getAttribute('name')})");
$functionId = $function->getId();
$functionInternalId = $function->getInternalId();
$this->migrateUsageMetrics("deployment.$functionId.storage.size", "function.$functionInternalId.deployments.storage");
$this->migrateUsageMetrics("builds.$functionId.compute.total", "$functionInternalId.builds");
$this->migrateUsageMetrics("builds.$functionId.compute.time", "$functionInternalId.builds.compute");
$this->migrateUsageMetrics("executions.$functionId.compute.total", "$functionInternalId.executions");
$this->migrateUsageMetrics("executions.$functionId.compute.time", "$functionInternalId.executions.compute");
}
}
/**
* Migrate Databases.
*
* @return void
* @throws \Exception
*/
private function migrateDatabases(): void
{
// Project level
$this->migrateUsageMetrics('databases.$all.count.total', 'databases');
$this->migrateUsageMetrics('collections.$all.count.total', 'collections');
$this->migrateUsageMetrics('documents.$all.count.total', 'documents');
foreach ($this->documentsIterator('databases') as $database) {
Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
$databaseTable = "database_{$database->getInternalId()}";
// Database level
$databaseId = $database->getId();
$databaseInternalId = $database->getInternalId();
$this->migrateUsageMetrics("collections.$databaseId.count.total", "$databaseInternalId.collections");
$this->migrateUsageMetrics("documents.$databaseId.count.total", "$databaseInternalId.documents");
foreach ($this->documentsIterator($databaseTable) as $collection) {
$collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})");
// Collection level
$collectionId = $collection->getId() ;
$collectionInternalId = $collection->getInternalId();
$this->migrateUsageMetrics("documents.$databaseId/$collectionId.count.total", "$databaseInternalId.$collectionInternalId.documents");
}
}
}
/**
* Migrate Collections.
*
* @return void
* @throws \Exception
*/
private function migrateCollections(): void
{
$internalProjectId = $this->project->getInternalId();
$collectionType = match ($internalProjectId) {
'console' => 'console',
default => 'projects',
};
$collections = $this->collections[$collectionType];
foreach ($collections as $collection) {
$id = $collection['$id'];
Console::log("Migrating Collection \"{$id}\"");
$this->projectDB->setNamespace("_$internalProjectId");
switch ($id) {
case 'stats':
try {
/**
* Delete 'type' attribute
*/
$this->projectDB->deleteAttribute($id, 'type');
/**
* Alter `signed` internal type on `value` attr
*/
$this->projectDB->updateAttribute($id, 'value', null, null, null, null, true);
$this->projectDB->deleteCachedCollection($id);
} catch (Throwable $th) {
Console::warning("'type' from {$id}: {$th->getMessage()}");
}
break;
}
}
}
/**
* Migrating all Bucket tables.
*
* @return void
* @throws \Exception
* @throws PDOException
*/
protected function migrateBuckets(): void
{
// Project level
$this->migrateUsageMetrics('buckets.$all.count.total', 'buckets');
$this->migrateUsageMetrics('files.$all.count.total', 'files');
$this->migrateUsageMetrics('files.$all.storage.size', 'files.storage');
// There is also project.$all.storage.size which is the same as files.$all.storage.size
foreach ($this->documentsIterator('buckets') as $bucket) {
$id = "bucket_{$bucket->getInternalId()}";
Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
// Bucket level
$bucketId = $bucket->getId();
$bucketInternalId = $bucket->getInternalId();
$this->migrateUsageMetrics("files.$bucketId.count.total", "$bucketInternalId.files");
$this->migrateUsageMetrics("files.$bucketId.storage.size", "$bucketInternalId.files.storage");
// some stats come with $ prefix in front of the id -> files.$650c3fda307b7fec4934.storage.size;
}
}
}

4
src/Appwrite/Platform/Services/Tasks.php
View file

@ -12,7 +12,6 @@ use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
@ -21,6 +20,7 @@ use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
use Appwrite\Platform\Tasks\GetMigrationStats;
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
use Appwrite\Platform\Tasks\CreateInfMetric;
class Tasks extends Service
{
@ -29,7 +29,6 @@ class Tasks extends Service
$this->type = self::TYPE_CLI;
$this
->addAction(Version::getName(), new Version())
->addAction(Usage::getName(), new Usage())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Hamster::getName(), new Hamster())
@ -46,6 +45,7 @@ class Tasks extends Service
->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
->addAction(GetMigrationStats::getName(), new GetMigrationStats())
->addAction(CreateInfMetric::getName(), new CreateInfMetric())
;
}

6
src/Appwrite/Platform/Services/Workers.php
View file

@ -13,6 +13,8 @@ use Appwrite\Platform\Workers\Functions;
use Appwrite\Platform\Workers\Builds;
use Appwrite\Platform\Workers\Deletes;
use Appwrite\Platform\Workers\Hamster;
use Appwrite\Platform\Workers\Usage;
use Appwrite\Platform\Workers\UsageHook;
use Appwrite\Platform\Workers\Migrations;
class Workers extends Service
@ -30,8 +32,10 @@ class Workers extends Service
->addAction(Functions::getName(), new Functions())
->addAction(Builds::getName(), new Builds())
->addAction(Deletes::getName(), new Deletes())
->addAction(Migrations::getName(), new Migrations())
->addAction(Hamster::getName(), new Hamster())
->addAction(UsageHook::getName(), new UsageHook())
->addAction(Usage::getName(), new Usage())
->addAction(Migrations::getName(), new Migrations())
;
}

530
src/Appwrite/Platform/Tasks/CalcTierStats.php
View file

@ -5,6 +5,7 @@ namespace Appwrite\Platform\Tasks;
use Exception;
use League\Csv\CannotInsertRecord;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
@ -15,6 +16,7 @@ use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
use Utopia\Validator\Text;
class CalcTierStats extends Action
{
@ -24,6 +26,7 @@ class CalcTierStats extends Action
private array $columns = [
'Project ID',
'Organization ID',
'Organization Email',
'Organization Members',
'Teams',
'Users',
@ -49,8 +52,9 @@ class CalcTierStats extends Action
protected string $date;
private array $usageStats = [
'project.$all.network.requests' => 'Requests',
'project.$all.network.bandwidth' => 'Bandwidth',
'network.requests' => 'Requests',
'network.inbound' => 'Inbound',
'network.outbound' => 'Outbound',
];
@ -64,270 +68,115 @@ class CalcTierStats extends Action
$this
->desc('Get stats for projects')
->param('after', '', new Text(36), 'After cursor', true)
->param('projectId', '', new Text(36), 'Select project to validate', true)
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('getProjectDB')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
->callback(function ($after, $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) {
$this->action($after, $projectId, $pools, $cache, $dbForConsole, $getProjectDB, $register);
});
}
/**
* @throws \Utopia\Exception
* @throws CannotInsertRecord
*/
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
public function action(string $after, string $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register): void
{
//docker compose exec -t appwrite calc-tier-stats
Console::title('Cloud free tier stats calculation V1');
Console::success(APP_NAME . ' cloud free tier stats calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
if (!empty($projectId)) {
try {
console::log("Project " . $projectId);
$project = $dbForConsole->getDocument('projects', $projectId);
$dbForProject = call_user_func($getProjectDB, $project);
$data = $this->getData($project, $dbForConsole, $dbForProject);
$csv->insertOne($data);
$this->sendMail($register);
$projects = [$console];
$count = 0;
$limit = 100;
$sum = 100;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
$stats['Organization ID'] = $project->getAttribute('teamId', null);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$stats['Organization Members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
]);
} else {
$stats['Organization Members'] = 0;
}
/** Get Total internal Teams */
try {
$stats['Teams'] = $dbForProject->count('teams', []);
} catch (\Throwable) {
$stats['Teams'] = 0;
}
/** Get Total users */
try {
$stats['Users'] = $dbForProject->count('users', []);
} catch (\Throwable) {
$stats['Users'] = 0;
}
/** Get Usage stats */
$range = '30d';
$periods = [
'30d' => [
'period' => '1d',
'limit' => 30,
]
];
$tmp = [];
$metrics = $this->usageStats;
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
foreach ($metrics as $metric => $name) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$tmp[$metric] = [];
foreach ($requestDocs as $requestDoc) {
if (empty($requestDoc)) {
continue;
}
$tmp[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$tmp[$metric] = array_reverse($tmp[$metric]);
$tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
}
});
foreach ($tmp as $key => $value) {
$stats[$metrics[$key]] = $value;
}
try {
/** Get Domains */
$stats['Domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Domains'] = 0;
}
try {
/** Get Api keys */
$stats['Api keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Api keys'] = 0;
}
try {
/** Get Webhooks */
$stats['Webhooks'] = $dbForConsole->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Webhooks'] = 0;
}
try {
/** Get Platforms */
$stats['Platforms'] = $dbForConsole->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Platforms'] = 0;
}
/** Get Files & Buckets */
$filesCount = 0;
$filesSum = 0;
$maxFileSize = 0;
$counter = 0;
try {
$buckets = $dbForProject->find('buckets', []);
foreach ($buckets as $bucket) {
$file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
if (empty($file)) {
continue;
}
$filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
$filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
$maxFileSize = $file->getAttribute('sizeOriginal');
}
$counter++;
}
} catch (\Throwable) {
;
}
$stats['Buckets'] = $counter;
$stats['Files'] = $filesCount;
$stats['Storage (bytes)'] = $filesSum;
$stats['Max File Size (bytes)'] = $maxFileSize;
try {
/** Get Total Functions */
$stats['Databases'] = $dbForProject->count('databases', []);
} catch (\Throwable) {
$stats['Databases'] = 0;
}
/** Get Total Functions */
try {
$stats['Functions'] = $dbForProject->count('functions', []);
} catch (\Throwable) {
$stats['Functions'] = 0;
}
/** Get Total Deployments */
try {
$stats['Deployments'] = $dbForProject->count('deployments', []);
} catch (\Throwable) {
$stats['Deployments'] = 0;
}
/** Get Total Executions */
try {
$stats['Executions'] = $dbForProject->count('executions', []);
} catch (\Throwable) {
$stats['Executions'] = 0;
}
/** Get Total Migrations */
try {
$stats['Migrations'] = $dbForProject->count('migrations', []);
} catch (\Throwable) {
$stats['Migrations'] = 0;
}
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
return;
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$queries = [];
$pools
->get('console')
->reclaim();
if (!empty($after)) {
Console::info("Iterating remaining projects after project with ID {$after}");
$project = $dbForConsole->getDocument('projects', $after);
$queries = [Query::cursorAfter($project)];
} else {
Console::info("Iterating all projects");
}
$this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole, $csv) {
$projectId = $project->getId();
console::log("Project " . $projectId);
try {
$dbForProject = call_user_func($getProjectDB, $project);
$data = $this->getData($project, $dbForConsole, $dbForProject);
$csv->insertOne($data);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
});
$this->sendMail($register);
}
private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
{
$limit = 1000;
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
if ($latestDocument != null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$results = $database->find('projects', $newQueries);
if (empty($results)) {
return;
}
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
$latestDocument = $results[array_key_last($results)];
}
}
private function sendMail(Registry $register): void
{
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
@ -339,7 +188,6 @@ class CalcTierStats extends Action
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
@ -356,4 +204,206 @@ class CalcTierStats extends Action
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
private function getData(Document $project, Database $dbForConsole, Database $dbForProject): array
{
$stats['Project ID'] = $project->getId();
$stats['Organization ID'] = $project->getAttribute('teamId', null);
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$teamInternalId]),
]);
if (!$membership || $membership->isEmpty()) {
Console::error('Membership not found. Skipping project : ' . $project->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$stats['Organization Email'] = $user->getAttribute('email', null);
}
} else {
Console::error("Email was not found for this Organization ID :{$teamInternalId}");
}
/** Get Total Members */
if ($teamInternalId) {
$stats['Organization Members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
]);
} else {
$stats['Organization Members'] = 0;
}
/** Get Total internal Teams */
try {
$stats['Teams'] = $dbForProject->count('teams', []);
} catch (\Throwable) {
$stats['Teams'] = 0;
}
/** Get Total users */
try {
$stats['Users'] = $dbForProject->count('users', []);
} catch (\Throwable) {
$stats['Users'] = 0;
}
/** Get Usage stats */
$range = '30d';
$periods = [
'30d' => [
'period' => '1d',
'limit' => 30,
]
];
$tmp = [];
$metrics = $this->usageStats;
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
foreach ($metrics as $metric => $name) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$tmp[$metric] = [];
foreach ($requestDocs as $requestDoc) {
if (empty($requestDoc)) {
continue;
}
$tmp[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$tmp[$metric] = array_reverse($tmp[$metric]);
$tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
}
});
foreach ($tmp as $key => $value) {
$stats[$metrics[$key]] = $value;
}
/**
* Workaround to combine network.inbound+network.outbound as network.
*/
$stats['Bandwidth'] = ($stats['Inbound'] ?? 0) + ($stats['Outbound'] ?? 0);
unset($stats['Inbound']);
unset($stats['Outbound']);
try {
/** Get Domains */
$stats['Domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Domains'] = 0;
}
try {
/** Get Api keys */
$stats['Api keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Api keys'] = 0;
}
try {
/** Get Webhooks */
$stats['Webhooks'] = $dbForConsole->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Webhooks'] = 0;
}
try {
/** Get Platforms */
$stats['Platforms'] = $dbForConsole->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Platforms'] = 0;
}
/** Get Files & Buckets */
$filesCount = 0;
$filesSum = 0;
$maxFileSize = 0;
$counter = 0;
try {
$buckets = $dbForProject->find('buckets', []);
foreach ($buckets as $bucket) {
$file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
if (empty($file)) {
continue;
}
$filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
$filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
$maxFileSize = $file->getAttribute('sizeOriginal');
}
$counter++;
}
} catch (\Throwable $t) {
Console::error("Error while counting buckets: {$project->getId()}");
}
$stats['Buckets'] = $counter;
$stats['Files'] = $filesCount;
$stats['Storage (bytes)'] = $filesSum;
$stats['Max File Size (bytes)'] = $maxFileSize;
try {
/** Get Total Functions */
$stats['Databases'] = $dbForProject->count('databases', []);
} catch (\Throwable) {
$stats['Databases'] = 0;
}
/** Get Total Functions */
try {
$stats['Functions'] = $dbForProject->count('functions', []);
} catch (\Throwable) {
$stats['Functions'] = 0;
}
/** Get Total Deployments */
try {
$stats['Deployments'] = $dbForProject->count('deployments', []);
} catch (\Throwable) {
$stats['Deployments'] = 0;
}
/** Get Total Executions */
try {
$stats['Executions'] = $dbForProject->count('executions', []);
} catch (\Throwable) {
$stats['Executions'] = 0;
}
/** Get Total Migrations */
try {
$stats['Migrations'] = $dbForProject->count('migrations', []);
} catch (\Throwable) {
$stats['Migrations'] = 0;
}
return array_values($stats);
}
}

413
src/Appwrite/Platform/Tasks/CreateInfMetric.php Normal file
View file

@ -0,0 +1,413 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Exception;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
use Utopia\Validator\Text;
class CreateInfMetric extends Action
{
public static function getName(): string
{
return 'create-inf-metric';
}
public function __construct()
{
$this
->desc('Create infinity stats metric')
->param('after', '', new Text(36), 'After cursor', true)
->param('projectId', '', new Text(36), 'Select project to validate', true)
->inject('getProjectDB')
->inject('dbForConsole')
->callback(function (string $after, string $projectId, callable $getProjectDB, Database $dbForConsole) {
$this->action($after, $projectId, $getProjectDB, $dbForConsole);
});
}
/**
* @throws Exception
* @throws Exception\Timeout
* @throws Exception\Query
*/
public function action(string $after, string $projectId, callable $getProjectDB, Database $dbForConsole): void
{
Console::title('Create infinity metric V1');
Console::success(APP_NAME . ' Create infinity metric started');
if (!empty($projectId)) {
try {
$project = $dbForConsole->getDocument('projects', $projectId);
$dbForProject = call_user_func($getProjectDB, $project);
$this->getUsageData($dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
} else {
$queries = [];
if (!empty($after)) {
Console::info("Iterating remaining projects after project with ID {$after}");
$project = $dbForConsole->getDocument('projects', $after);
$queries = [Query::cursorAfter($project)];
} else {
Console::info("Iterating all projects");
}
$this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB) {
$projectId = $project->getId();
try {
$dbForProject = call_user_func($getProjectDB, $project);
$this->getUsageData($dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
});
}
}
/**
* @param Database $database
* @param string $collection
* @param array $queries
* @param callable|null $callback
* @return void
* @throws Exception
* @throws Exception\Query
* @throws Exception\Timeout
*/
private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
{
$limit = 1000;
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
if ($latestDocument != null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$results = $database->find($collection, $newQueries);
if (empty($results)) {
return;
}
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
$latestDocument = $results[array_key_last($results)];
}
}
/**
* @param Database $dbForProject
* @param Document $project
* @return void
*/
private function getUsageData(Database $dbForProject, Document $project): void
{
try {
$this->network($dbForProject);
$this->sessions($dbForProject);
$this->users($dbForProject);
$this->teams($dbForProject);
$this->databases($dbForProject);
$this->functions($dbForProject);
$this->storage($dbForProject);
} catch (\Throwable $th) {
var_dump($th->getMessage());
}
Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
}
/**
* @param Database $dbForProject
* @param string $metric
* @param int|float $value
* @return void
* @throws Exception
* @throws Exception\Authorization
* @throws Exception\Conflict
* @throws Exception\Restricted
* @throws Exception\Structure
*/
private function createInfMetric(database $dbForProject, string $metric, int|float $value): void
{
try {
$id = \md5("_inf_{$metric}");
$dbForProject->deleteDocument('stats_v2', $id);
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'metric' => $metric,
'period' => 'inf',
'value' => (int)$value,
'time' => null,
'region' => 'default',
]));
} catch (Duplicate $th) {
console::log("Error while creating inf metric: duplicate id {$metric} {$id}");
}
}
/**
* @param Database $dbForProject
* @param string $metric
* @return int|float
* @throws Exception
*/
protected function getFromMetric(database $dbForProject, string $metric): int|float
{
return $dbForProject->sum('stats_v2', 'value', [
Query::equal('metric', [
$metric,
]),
Query::equal('period', ['1d']),
]);
}
/**
* @param Database $dbForProject
* @throws Exception
* @throws Exception\Authorization
* @throws Exception\Conflict
* @throws Exception\Restricted
* @throws Exception\Structure
*/
private function network(database $dbForProject)
{
$this->createInfMetric($dbForProject, 'network.inbound', $this->getFromMetric($dbForProject, 'network.inbound'));
$this->createInfMetric($dbForProject, 'network.outbound', $this->getFromMetric($dbForProject, 'network.outbound'));
$this->createInfMetric($dbForProject, 'network.requests', $this->getFromMetric($dbForProject, 'network.requests'));
}
/**
* @throws Exception\Authorization
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception\Timeout
* @throws Exception\Structure
* @throws Exception
* @throws Exception\Query
*/
private function storage(database $dbForProject)
{
$bucketsCount = 0;
$filesCount = 0;
$filesStorageSum = 0;
$buckets = $dbForProject->find('buckets');
foreach ($buckets as $bucket) {
$files = $dbForProject->count('bucket_' . $bucket->getInternalId());
$this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $files);
$filesStorage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal');
$this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStorage);
$bucketsCount++;
$filesCount += $files;
$filesStorageSum += $filesStorage;
}
$this->createInfMetric($dbForProject, 'buckets', $bucketsCount);
$this->createInfMetric($dbForProject, 'files', $filesCount);
$this->createInfMetric($dbForProject, 'files.storage', $filesStorageSum);
}
/**
* @throws Exception\Authorization
* @throws Exception\Timeout
* @throws Exception\Restricted
* @throws Exception\Structure
* @throws Exception\Conflict
* @throws Exception
* @throws Exception\Query
*/
private function functions(Database $dbForProject)
{
$functionsCount = 0;
$deploymentsCount = 0;
$buildsCount = 0;
$buildsStorageSum = 0;
$buildsComputeSum = 0;
$executionsCount = 0;
$executionsComputeSum = 0;
$deploymentsStorageSum = 0;
//functions
$functions = $dbForProject->find('functions');
foreach ($functions as $function) {
//deployments
$deployments = $dbForProject->find('deployments', [
Query::equal('resourceType', ['functions']),
Query::equal('resourceInternalId', [$function->getInternalId()]),
]);
$deploymentCount = 0;
$deploymentStorageSum = 0;
foreach ($deployments as $deployment) {
//builds
$builds = $dbForProject->count('builds', [
Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
]);
$buildsCompute = $dbForProject->sum('builds', 'duration', [
Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
]);
$buildsStorage = $dbForProject->sum('builds', 'size', [
Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
]);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.builds', $builds);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.storage', $buildsCompute * 1000);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildsStorage);
$buildsCount += $builds;
$buildsComputeSum += $buildsCompute;
$buildsStorageSum += $buildsStorage;
$deploymentCount++;
$deploymentsCount++;
$deploymentsStorageSum += $deployment['size'];
$deploymentStorageSum += $deployment['size'];
}
$this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments', $deploymentCount);
$this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments.storage', $deploymentStorageSum);
//executions
$executions = $dbForProject->count('executions', [
Query::equal('functionInternalId', [$function->getInternalId()]),
]);
$executionsCompute = $dbForProject->sum('executions', 'duration', [
Query::equal('functionInternalId', [$function->getInternalId()]),
]);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executions);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', $executionsCompute * 1000);
$executionsCount += $executions;
$executionsComputeSum += $executionsCompute;
$functionsCount++;
}
$this->createInfMetric($dbForProject, 'functions', $functionsCount);
$this->createInfMetric($dbForProject, 'deployments', $deploymentsCount);
$this->createInfMetric($dbForProject, 'deployments.storage', $deploymentsStorageSum);
$this->createInfMetric($dbForProject, 'builds', $buildsCount);
$this->createInfMetric($dbForProject, 'builds.compute', $buildsComputeSum * 1000);
$this->createInfMetric($dbForProject, 'builds.storage', $buildsStorageSum);
$this->createInfMetric($dbForProject, 'executions', $executionsCount);
$this->createInfMetric($dbForProject, 'executions.compute', $executionsComputeSum * 1000);
}
/**
* @throws Exception\Authorization
* @throws Exception\Timeout
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
* @throws Exception\Query
*/
private function databases(Database $dbForProject)
{
$databasesCount = 0;
$collectionsCount = 0;
$documentsCount = 0;
$databases = $dbForProject->find('databases');
foreach ($databases as $database) {
$collectionCount = 0;
$collections = $dbForProject->find('database_' . $database->getInternalId());
foreach ($collections as $collection) {
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$this->createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documents);
$documentsCount += $documents;
$collectionCount++;
$collectionsCount++;
}
$this->createInfMetric($dbForProject, $database->getInternalId() . '.collections', $collectionCount);
$this->createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCount);
$databasesCount++;
}
$this->createInfMetric($dbForProject, 'collections', $collectionsCount);
$this->createInfMetric($dbForProject, 'databases', $databasesCount);
$this->createInfMetric($dbForProject, 'documents', $documentsCount);
}
/**
* @throws Exception\Authorization
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
*/
private function users(Database $dbForProject)
{
$users = $dbForProject->count('users');
$this->createInfMetric($dbForProject, 'users', $users);
}
/**
* @throws Exception\Authorization
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
*/
private function sessions(Database $dbForProject)
{
$users = $dbForProject->count('sessions');
$this->createInfMetric($dbForProject, 'sessions', $users);
}
/**
* @throws Exception\Authorization
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
*/
private function teams(Database $dbForProject)
{
$teams = $dbForProject->count('teams');
$this->createInfMetric($dbForProject, 'teams', $teams);
}
}

57
src/Appwrite/Platform/Tasks/Maintenance.php
View file

@ -36,50 +36,77 @@ class Maintenance extends Action
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
$auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
$abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes);
$this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes);
$this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$queueForDeletes->setProject($project);
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
});
$this->notifyDeleteConnections($queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
$this->renewCertificates($dbForConsole, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
}, $interval);
}
private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void
protected function foreachProject(Database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
{
($queueForDeletes)
->setType(DELETE_TYPE_EXECUTIONS)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void
private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void
{
($queueForDeletes)
->setType(DELETE_TYPE_ABUSE)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void
private function notifyDeleteAuditLogs(Delete $queueForDeletes): void
{
($queueForDeletes)
->setType(DELETE_TYPE_AUDIT)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}

60
src/Appwrite/Platform/Tasks/Usage.php
View file

@ -1,60 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Usage\Calculators\TimeSeries;
use InfluxDB\Database as InfluxDatabase;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Throwable;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
class Usage extends Action
{
public static function getName(): string
{
return 'usage';
}
public function __construct()
{
$this
->desc('Schedules syncing data from influxdb to Appwrite console db')
->inject('dbForConsole')
->inject('influxdb')
->inject('register')
->inject('getProjectDB')
->inject('logError')
->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
}
protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{
}
public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
$loopStart = microtime(true);
$usage->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
}
}

38
src/Appwrite/Platform/Workers/Builds.php
View file

@ -6,7 +6,6 @@ use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
use Exception;
@ -48,11 +47,11 @@ class Builds extends Action
->inject('dbForConsole')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('usage')
->inject('queueForUsage')
->inject('cache')
->inject('dbForProject')
->inject('getFunctionsDevice')
->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice));
->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $queueForUsage, $cache, $dbForProject, $getFunctionsDevice));
}
/**
@ -60,14 +59,14 @@ class Builds extends Action
* @param Database $dbForConsole
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Stats $usage
* @param Usage $queueForUsage
* @param Cache $cache
* @param Database $dbForProject
* @param callable $getFunctionsDevice
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void
public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void
{
$payload = $message->getPayload() ?? [];
@ -86,7 +85,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $usage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template);
$this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template);
break;
default:
@ -98,7 +97,7 @@ class Builds extends Action
* @param callable $getFunctionsDevice
* @param Func $queueForFunctions
* @param Event $queueForEvents
* @param Stats $usage
* @param Usage $queueForUsage
* @param Database $dbForConsole
* @param Database $dbForProject
* @param GitHub $github
@ -110,7 +109,7 @@ class Builds extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void
protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void
{
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
@ -529,19 +528,16 @@ class Builds extends Action
roles: $target['roles']
);
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('functionId', $function->getId())
->setParam('builds.{scope}.compute', 1)
->setParam('buildStatus', $build->getAttribute('status', ''))
->setParam('buildTime', $build->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->trigger();
}
}

2
src/Appwrite/Platform/Workers/Certificates.php
View file

@ -430,7 +430,7 @@ class Certificates extends Action
$message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl');
$message
->setParam('{{body}}', $locale->getText("emails.certificate.body"))
->setParam('{{body}}', $locale->getText("emails.certificate.body"), escapeHtml: false)
->setParam('{{hello}}', $locale->getText("emails.certificate.hello"))
->setParam('{{footer}}', $locale->getText("emails.certificate.footer"))
->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks"))

146
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Executor\Executor;
use Throwable;
use Utopia\Abuse\Abuse;
@ -45,14 +46,17 @@ class Deletes extends Action
->inject('getFunctionsDevice')
->inject('getBuildsDevice')
->inject('getCacheDevice')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice));
->inject('abuseRetention')
->inject('executionRetention')
->inject('auditRetention')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention));
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention): void
{
$payload = $message->getPayload() ?? [];
@ -114,12 +118,12 @@ class Deletes extends Action
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $datetime);
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
break;
case DELETE_TYPE_AUDIT:
if (!empty($datetime)) {
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $datetime);
if (!$project->isEmpty()) {
$this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
}
if (!$document->isEmpty()) {
@ -127,7 +131,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_ABUSE:
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $datetime);
$this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
break;
case DELETE_TYPE_REALTIME:
@ -135,10 +139,10 @@ class Deletes extends Action
break;
case DELETE_TYPE_SESSIONS:
$this->deleteExpiredSessions($dbForConsole, $getProjectDB);
$this->deleteExpiredSessions($project, $getProjectDB);
break;
case DELETE_TYPE_USAGE:
$this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime);
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project, $getProjectDB, $resource);
@ -337,16 +341,14 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) {
$dbForProject = $getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
});
$dbForProject = $getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats_v2', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
}
/**
@ -387,6 +389,7 @@ class Deletes extends Action
*/
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void
{
$projects = $dbForConsole->find('projects', [
Query::equal('teamInternalId', [$document->getInternalId()])
]);
@ -542,15 +545,13 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
$dbForProject = $getProjectDB($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
});
$dbForProject = $getProjectDB($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
}
/**
@ -559,20 +560,16 @@ class Deletes extends Action
* @return void
* @throws Exception|Throwable
*/
private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void
private function deleteExpiredSessions(Document $project, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) {
$dbForProject = $getProjectDB($project);
$project = $dbForConsole->getDocument('projects', $project->getId());
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
});
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
}
/**
@ -596,22 +593,16 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
{
if (empty($datetime)) {
throw new Exception('Failed to delete audit logs. No datetime provided');
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($abuseRetention);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
});
}
/**
@ -621,21 +612,15 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void
{
if (empty($datetime)) {
throw new Exception('Failed to delete audit logs. No datetime provided');
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($auditRetention);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
});
}
/**
@ -870,39 +855,6 @@ class Deletes extends Action
}
}
/**
* @param Database $dbForConsole
* @param callable $callback
* @throws Exception
*/
private function deleteForProjectIds(database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
/**
* @param string $collection collectionID
* @param array $queries

45
src/Appwrite/Platform/Workers/Functions.php
View file

@ -2,9 +2,9 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Usage\Stats;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Utopia\Response\Model\Execution;
use Exception;
@ -39,13 +39,14 @@ class Functions extends Action
{
$this
->desc('Functions worker')
->groups(['functions'])
->inject('message')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForEvents')
->inject('usage')
->inject('queueForUsage')
->inject('log')
->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $log));
->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log));
}
/**
@ -53,7 +54,7 @@ class Functions extends Action
* @param Database $dbForProject
* @param Func $queueForFunctions
* @param Event $queueForEvents
* @param Stats $usage
* @param Usage $queueForUsage
* @param Log $log
* @return void
* @throws Authorization
@ -61,7 +62,7 @@ class Functions extends Action
* @throws \Utopia\Database\Exception
* @throws Conflict
*/
public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log): void
public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log): void
{
$payload = $message->getPayload() ?? [];
@ -117,7 +118,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
usage: $usage,
queueForUsage: $queueForUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@ -153,7 +154,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
usage: $usage,
queueForUsage: $queueForUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@ -174,7 +175,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
usage: $usage,
queueForUsage: $queueForUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@ -197,7 +198,7 @@ class Functions extends Action
* @param Log $log
* @param Database $dbForProject
* @param Func $queueForFunctions
* @param Stats $usage
* @param Usage $queueForUsage
* @param Event $queueForEvents
* @param Document $project
* @param Document $function
@ -221,7 +222,7 @@ class Functions extends Action
Log $log,
Database $dbForProject,
Func $queueForFunctions,
stats $usage,
Usage $queueForUsage,
Event $queueForEvents,
Document $project,
Document $function,
@ -420,6 +421,16 @@ class Functions extends Action
$error = $th->getMessage();
$errorCode = $th->getCode();
} finally {
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
}
if ($function->getAttribute('logging')) {
@ -471,19 +482,5 @@ class Functions extends Action
if (!empty($error)) {
throw new Exception($error, $errorCode);
}
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$usage
->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
}
}

90
src/Appwrite/Platform/Workers/Hamster.php
View file

@ -22,17 +22,18 @@ use Utopia\Pools\Group;
class Hamster extends Action
{
private array $metrics = [
'usage_files' => 'files.$all.count.total',
'usage_buckets' => 'buckets.$all.count.total',
'usage_databases' => 'databases.$all.count.total',
'usage_documents' => 'documents.$all.count.total',
'usage_collections' => 'collections.$all.count.total',
'usage_storage' => 'project.$all.storage.size',
'usage_requests' => 'project.$all.network.requests',
'usage_bandwidth' => 'project.$all.network.bandwidth',
'usage_users' => 'users.$all.count.total',
'usage_sessions' => 'sessions.email.requests.create',
'usage_executions' => 'executions.$all.compute.total',
'usage_files' => 'files',
'usage_buckets' => 'buckets',
'usage_databases' => 'databases',
'usage_documents' => 'documents',
'usage_collections' => 'collections',
'usage_storage' => 'files.storage',
'usage_requests' => 'network.requests',
'usage_inbound' => 'network.inbound',
'usage_outbound' => 'network.outbound',
'usage_users' => 'users',
'usage_sessions' => 'sessions',
'usage_executions' => 'executions',
];
protected Mixpanel $mixpanel;
@ -217,6 +218,15 @@ class Hamster extends Action
}
}
/** Add billing information to the project */
$organization = $dbForConsole->findOne('teams', [
Query::equal('$internalId', [$teamInternalId])
]);
$billing = $this->getBillingDetails($organization);
$statsPerProject['billing_plan'] = $billing['billing_plan'] ?? null;
$statsPerProject['billing_start_date'] = $billing['billing_start_date'] ?? null;
/** Get Domains */
$statsPerProject['custom_domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
@ -276,7 +286,7 @@ class Hamster extends Action
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
$requestDocs = $dbForProject->find('stats_v2', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
@ -298,6 +308,17 @@ class Hamster extends Action
}
});
/**
* Workaround to combine network.Inbound+network.outbound as bandwidth.
*/
$statsPerProject["usage_bandwidth_infinity"] = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"];
$statsPerProject["usage_bandwidth_24h"] = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"];
unset($statsPerProject["usage_outbound_24h"]);
unset($statsPerProject["usage_inbound_24h"]);
unset($statsPerProject["usage_outbound_infinity"]);
unset($statsPerProject["usage_inbound_infinity"]);
if (isset($statsPerProject['email'])) {
/** Send data to mixpanel */
$res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
@ -332,7 +353,6 @@ class Hamster extends Action
/**
* @param Document $organization
* @param Database $dbForConsole
* @throws \Utopia\Database\Exception
*/
private function getStatsForOrganization(Document $organization, Database $dbForConsole): void
{
@ -346,22 +366,26 @@ class Hamster extends Action
/** Organization name */
$statsPerOrganization['name'] = $organization->getAttribute('name');
/** Get Email and of the organization owner */
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$organization->getInternalId()]),
]);
if (!$membership || $membership->isEmpty()) {
throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$statsPerOrganization['email'] = $user->getAttribute('email', null);
}
/** Add billing information */
$billing = $this->getBillingDetails($organization);
$statsPerOrganization['billing_plan'] = $billing['billing_plan'] ?? null;
$statsPerOrganization['billing_start_date'] = $billing['billing_start_date'] ?? null;
$statsPerOrganization['marked_for_deletion'] = $billing['markedForDeletion'] ?? 0;
$statsPerOrganization['billing_plan_downgrade'] = $billing['billing_plan_downgrade'] ?? null;
/** Organization Creation Date */
$statsPerOrganization['created'] = $organization->getAttribute('$createdAt');
@ -400,6 +424,16 @@ class Hamster extends Action
$statsPerUser['time'] = $user->getAttribute('$time');
/** Add billing information */
$organization = $dbForConsole->findOne('teams', [
Query::equal('userInternalId', [$user->getInternalId()])
]);
$billing = $this->getBillingDetails($organization);
$statsPerUser['billing_plan'] = $billing['billing_plan'] ?? null;
$statsPerUser['billing_start_date'] = $billing['billing_start_date'] ?? null;
/** Organization name */
$statsPerUser['name'] = $user->getAttribute('name');
@ -434,4 +468,28 @@ class Hamster extends Action
Console::error($e->getMessage());
}
}
private function getBillingDetails(bool|Document $team): array
{
$billing = [];
if (!empty($team) && !$team->isEmpty()) {
$billingPlan = $team->getAttribute('billingPlan', null);
$billingPlanDowngrade = $team->getAttribute('billingPlanDowngrade', null);
if (!empty($billingPlan) && empty($billingPlanDowngrade)) {
$billing['billing_plan'] = $billingPlan;
}
if (in_array($billingPlan, ['tier-1', 'tier-2'])) {
$billingStartDate = $team->getAttribute('billingStartDate', null);
$billing['billing_start_date'] = $billingStartDate;
}
$billing['marked_for_deletion'] = $team->getAttribute('markedForDeletion', 0);
$billing['billing_plan_downgrade'] = $billingPlanDowngrade;
}
return $billing;
}
}

21
src/Appwrite/Platform/Workers/Mails.php
View file

@ -59,11 +59,16 @@ class Mails extends Action
$variables = $payload['variables'];
$name = $payload['name'];
$body = $payload['body'];
$bodyTemplate = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl');
$bodyTemplate->setParam('{{body}}', $body);
$attachment = $payload['attachment'] ?? [];
$bodyTemplate = $payload['bodyTemplate'];
if (empty($bodyTemplate)) {
$bodyTemplate = __DIR__ . '/../../../../app/config/locale/templates/email-base.tpl';
}
$bodyTemplate = Template::fromFile($bodyTemplate);
$bodyTemplate->setParam('{{body}}', $body, escapeHtml: false);
foreach ($variables as $key => $value) {
$bodyTemplate->setParam('{{' . $key . '}}', $value);
// TODO: hotfix for redirect param
$bodyTemplate->setParam('{{' . $key . '}}', $value, escapeHtml: $key !== 'redirect');
}
$body = $bodyTemplate->render();
@ -89,6 +94,14 @@ class Mails extends Action
$mail->Subject = $subject;
$mail->Body = $body;
$mail->AltBody = \strip_tags($body);
if (!empty($attachment['content'] ?? '')) {
$mail->AddStringAttachment(
base64_decode($attachment['content']),
$attachment['filename'] ?? 'unknown.file',
$attachment['encoding'] ?? PHPMailer::ENCODING_BASE64,
$attachment['type'] ?? 'plain/text'
);
}
try {
$mail->send();

238
src/Appwrite/Platform/Workers/Usage.php Normal file
View file

@ -0,0 +1,238 @@
<?php
namespace Appwrite\Platform\Workers;
use Exception;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
class Usage extends Action
{
protected static array $stats = [];
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
protected const INFINITY_PERIOD = '_inf_';
protected const DEBUG_PROJECT_ID = 85293;
public static function getName(): string
{
return 'usage';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Usage worker')
->inject('message')
->inject('getProjectDB')
->callback(function (Message $message, callable $getProjectDB) {
$this->action($message, $getProjectDB);
});
}
/**
* @param Message $message
* @param callable $getProjectDB
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, callable $getProjectDB): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$this->reduce(
project: $project,
document: new Document($document),
metrics: $payload['metrics'],
getProjectDB: $getProjectDB
);
}
if ($project->getInternalId() == self::DEBUG_PROJECT_ID) {
var_dump([
'type' => 'payload',
'project' => $project->getInternalId(),
'database' => $project['database'] ?? '',
$payload['metrics']
]);
var_dump('==========================');
}
self::$stats[$projectId]['project'] = $project;
foreach ($payload['metrics'] ?? [] as $metric) {
if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) {
self::$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
self::$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
}
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.
* @param Document $project
* @param Document $document
* @param array $metrics
* @param callable $getProjectDB
* @return void
*/
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
try {
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executions['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Exception $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
}
}
}

138
src/Appwrite/Platform/Workers/UsageHook.php Normal file
View file

@ -0,0 +1,138 @@
<?php
namespace Appwrite\Platform\Workers;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Swoole\Timer;
use Utopia\Database\DateTime;
class UsageHook extends Usage
{
public static function getName(): string
{
return 'usageHook';
}
public function __construct()
{
$this
->setType(Action::TYPE_WORKER_START)
->inject('register')
->inject('getProjectDB')
->callback(function ($register, callable $getProjectDB) {
$this->action($register, $getProjectDB);
})
;
}
/**
* @param $register
* @param $getProjectDB
* @return void
*/
public function action($register, $getProjectDB): void
{
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000');
Timer::tick($interval, function () use ($register, $getProjectDB) {
$offset = count(self::$stats);
$projects = array_slice(self::$stats, 0, $offset, true);
array_splice(self::$stats, 0, $offset);
foreach ($projects as $data) {
$numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0;
$projectInternalId = $data['project']->getInternalId();
$database = $data['project']['database'] ?? '';
console::warning('Ticker started ' . DateTime::now());
if ($numberOfKeys === 0) {
continue;
}
try {
$dbForProject = $getProjectDB($data['project']);
if ($projectInternalId == 85293) {
var_dump([
'project' => $projectInternalId,
'database' => $database,
'time' => DateTime::now(),
'data' => $data['keys']
]);
}
foreach ($data['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
if ($projectInternalId == self::DEBUG_PROJECT_ID) {
var_dump([
'type' => 'create',
'period' => $period,
'metric' => $key,
'id' => $id,
'value' => $value
]);
}
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
if ($projectInternalId == self::DEBUG_PROJECT_ID) {
var_dump([
'type' => 'decrease',
'period' => $period,
'metric' => $key,
'id' => $id,
'value' => $value
]);
}
$dbForProject->decreaseDocumentAttribute(
'stats_v2',
$id,
'value',
abs($value)
);
} else {
if ($projectInternalId == self::DEBUG_PROJECT_ID) {
var_dump([
'type' => 'increase',
'period' => $period,
'metric' => $key,
'id' => $id,
'value' => $value
]);
}
$dbForProject->increaseDocumentAttribute(
'stats_v2',
$id,
'value',
$value
);
}
}
}
}
} catch (\Exception $e) {
console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage());
}
}
});
}
}

15
src/Appwrite/Usage/Calculator.php
View file

@ -1,15 +0,0 @@
<?php
namespace Appwrite\Usage;
abstract class Calculator
{
protected string $region;
public function __construct(string $region)
{
$this->region = $region;
}
abstract public function collect(): void;
}

560
src/Appwrite/Usage/Calculators/TimeSeries.php
View file

@ -1,560 +0,0 @@
<?php
namespace Appwrite\Usage\Calculators;
use Utopia\App;
use Appwrite\Usage\Calculator;
use Utopia\Database\Database;
use Utopia\Database\Document;
use InfluxDB\Database as InfluxDatabase;
use DateTime;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
class TimeSeries extends Calculator
{
/**
* InfluxDB
*
* @var InfluxDatabase
*/
protected InfluxDatabase $influxDB;
/**
* Utopia Database
*
* @var Database
*/
protected Database $database;
/**
* Error Handler Callback
*
* @var callable
*/
protected $errorHandler;
/**
* Callback to get project DB
*
* @var callable
*/
protected mixed $getProjectDB;
/**
* Registry
*
* @var Registry
*/
protected Registry $register;
/**
* Latest times for metric that was synced to the database
*
* @var array
*/
private array $latestTime = [];
/**
* Periods the metrics are collected for
* @var array
*/
protected array $periods = [
[
'key' => '1h',
'startTime' => '-24 hours'
],
[
'key' => '1d',
'startTime' => '-30 days'
]
];
/**
* All the metrics that we are collecting
*
* @var array
*/
protected array $metrics = [
'project.$all.network.requests' => [
'table' => 'appwrite_usage_project_{scope}_network_requests',
],
'project.$all.network.bandwidth' => [
'table' => 'appwrite_usage_project_{scope}_network_bandwidth',
],
'project.$all.network.inbound' => [
'table' => 'appwrite_usage_project_{scope}_network_inbound',
],
'project.$all.network.outbound' => [
'table' => 'appwrite_usage_project_{scope}_network_outbound',
],
/* Users service metrics */
'users.$all.requests.create' => [
'table' => 'appwrite_usage_users_{scope}_requests_create',
],
'users.$all.requests.read' => [
'table' => 'appwrite_usage_users_{scope}_requests_read',
],
'users.$all.requests.update' => [
'table' => 'appwrite_usage_users_{scope}_requests_update',
],
'users.$all.requests.delete' => [
'table' => 'appwrite_usage_users_{scope}_requests_delete',
],
'databases.$all.requests.create' => [
'table' => 'appwrite_usage_databases_{scope}_requests_create',
],
'databases.$all.requests.read' => [
'table' => 'appwrite_usage_databases_{scope}_requests_read',
],
'databases.$all.requests.update' => [
'table' => 'appwrite_usage_databases_{scope}_requests_update',
],
'databases.$all.requests.delete' => [
'table' => 'appwrite_usage_databases_{scope}_requests_delete',
],
'collections.$all.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
],
'collections.$all.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
],
'collections.$all.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
],
'collections.$all.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
],
'documents.$all.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
],
'documents.$all.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
],
'documents.$all.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
],
'documents.$all.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
],
'collections.databaseId.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId/collectionId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId', 'collectionId'],
],
'buckets.$all.requests.create' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_create',
],
'buckets.$all.requests.read' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_read',
],
'buckets.$all.requests.update' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_update',
],
'buckets.$all.requests.delete' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_delete',
],
'files.$all.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
],
'files.$all.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
],
'files.$all.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
],
'files.$all.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
],
'files.bucketId.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
'groupBy' => ['bucketId'],
],
'sessions.$all.requests.create' => [
'table' => 'appwrite_usage_sessions__{scope}_requests_create',
],
'sessions.provider.requests.create' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_create',
'groupBy' => ['provider'],
],
'sessions.$all.requests.delete' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_delete',
],
'executions.$all.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
],
'builds.$all.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
],
'executions.$all.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.$all.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'executions.$all.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'builds.$all.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'executions.functionId.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
],
'executions.functionId.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.functionId.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'failed',
],
],
'executions.functionId.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'success',
],
],
'builds.functionId.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'success',
],
],
// counters
'users.$all.count.total' => [
'table' => 'appwrite_usage_users_{scope}_count_total',
],
'buckets.$all.count.total' => [
'table' => 'appwrite_usage_buckets_{scope}_count_total',
],
'files.$all.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
],
'files.bucketId.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
'groupBy' => ['bucketId']
],
'databases.$all.count.total' => [
'table' => 'appwrite_usage_databases_{scope}_count_total',
],
'collections.$all.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
],
'documents.$all.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
],
'collections.databaseId.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId/collectionId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId', 'collectionId']
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size',
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size',
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
],
'files.$bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
],
'builds.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.functionId.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.time' => [
'table' => 'appwrite_usage_builds_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'project.$all.compute.time' => [ // Built time + execution time
'table' => 'appwrite_usage_project_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size'
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size'
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size'
],
'files.bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
]
];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null)
{
parent::__construct($region);
$this->database = $database;
$this->influxDB = $influxDB;
$this->getProjectDB = $getProjectDB;
$this->register = $register;
$this->errorHandler = $errorHandler;
}
/**
* Create or Update Mertic
* Create or update each metric in the stats collection for the given project
*
* @param string $projectId
* @param int $time
* @param string $period
* @param string $metric
* @param int $value
* @param int $type
*
* @return void
*/
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$project = $this->database->getDocument('projects', $projectId);
$database = call_user_func($this->getProjectDB, $project);
Authorization::skip(function () use ($database, $id, $period, $time, $metric, $value, $type, $projectId) {
try {
$document = $database->getDocument('stats', $id);
if ($document->isEmpty()) {
$database->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $metric,
'value' => $value,
'type' => $type,
'region' => $this->region,
]));
} else {
$database->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $value)
);
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
} else {
throw $e;
}
}
});
$this->register->get('pools')->reclaim();
}
/**
* Sync From InfluxDB
* Sync stats from influxDB to stats collection in the Appwrite database
*
* @param string $metric
* @param array $options
* @param array $period
*
* @return void
*/
private function syncFromInfluxDB(string $metric, array $options, array $period): void
{
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($this->latestTime[$metric][$period['key']])) {
$start = $this->latestTime[$metric][$period['key']];
}
$end = (new DateTime())->format(DateTime::RFC3339);
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
} else {
$filters = '';
}
$query = "SELECT sum(value) AS \"value\" ";
$query .= "FROM \"{$table}\" ";
$query .= "WHERE \"time\" > '{$start}' ";
$query .= "AND \"time\" < '{$end}' ";
$query .= "AND \"metric_type\"='counter' {$filters} ";
$query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} ";
$query .= "FILL(null)";
try {
$result = $this->influxDB->query($query);
$points = $result->getPoints();
foreach ($points as $point) {
$projectId = $point['projectId'];
if (!empty($projectId) && $projectId !== 'console') {
$metricUpdated = $metric;
if (!empty($groupBy)) {
foreach ($options['groupBy'] as $groupBy) {
$groupedBy = $point[$groupBy] ?? '';
if (empty($groupedBy)) {
continue;
}
$metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated);
}
}
$value = (!empty($point['value'])) ? $point['value'] : 0;
$this->createOrUpdateMetric(
$point['projectId'],
$point['time'],
$period['key'],
$metricUpdated,
$value,
0
);
$this->latestTime[$metric][$period['key']] = $point['time'];
}
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb");
} else {
throw $e;
}
}
}
/**
* Collect Stats
* Collect all the stats from Influd DB to Database
*
* @return void
*/
public function collect(): void
{
foreach ($this->periods as $period) {
foreach ($this->metrics as $metric => $options) { //for each metrics
try {
$this->syncFromInfluxDB($metric, $options, $period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
}
}
}
}
}

225
src/Appwrite/Usage/Stats.php
View file

@ -1,225 +0,0 @@
<?php
namespace Appwrite\Usage;
use Utopia\App;
class Stats
{
/**
* @var array
*/
protected $params = [];
/**
* @var mixed
*/
protected $statsd;
/**
* @var string
*/
protected $namespace = 'appwrite.usage';
/**
* Event constructor.
*
* @param mixed $statsd
*/
public function __construct($statsd)
{
$this->statsd = $statsd;
}
/**
* @param string $key
* @param mixed $value
*
* @return $this
*/
public function setParam(string $key, $value): self
{
$this->params[$key] = $value;
return $this;
}
/**
* @param string $key
*
* @return mixed|null
*/
public function getParam(string $key)
{
return (isset($this->params[$key])) ? $this->params[$key] : null;
}
/**
* @param string $namespace
*
* @return $this
*/
public function setNamespace(string $namespace): self
{
$this->namespace = $namespace;
return $this;
}
/**
* @return string
*/
public function getNamespace()
{
return $this->namespace;
}
/**
* Submit data to StatsD.
* Send various metrics to StatsD based on the parameters that are set
* @return void
*/
public function submit(): void
{
$projectId = $this->params['projectId'] ?? '';
$projectInternalId = $this->params['projectInternalId'];
$tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
$httpRequest = $this->params['project.{scope}.network.requests'] ?? 0;
$httpMethod = $this->params['httpMethod'] ?? '';
if ($httpRequest >= 1) {
$this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod));
}
$inbound = $this->params['project.{scope}.network.inbound'] ?? 0;
$outbound = $this->params['project.{scope}.network.outbound'] ?? 0;
$this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound);
$this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound);
$this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);
$usersMetrics = [
'users.{scope}.requests.create',
'users.{scope}.requests.read',
'users.{scope}.requests.update',
'users.{scope}.requests.delete',
'users.{scope}.count.total',
];
foreach ($usersMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$this->statsd->count($metric . $tags, $value);
}
}
$dbMetrics = [
'databases.{scope}.requests.create',
'databases.{scope}.requests.read',
'databases.{scope}.requests.update',
'databases.{scope}.requests.delete',
'collections.{scope}.requests.create',
'collections.{scope}.requests.read',
'collections.{scope}.requests.update',
'collections.{scope}.requests.delete',
'documents.{scope}.requests.create',
'documents.{scope}.requests.read',
'documents.{scope}.requests.update',
'documents.{scope}.requests.delete',
'databases.{scope}.count.total',
'collections.{scope}.count.total',
'documents.{scope}.count.total'
];
foreach ($dbMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
$this->statsd->count($metric . $dbTags, $value);
}
}
$storageMertics = [
'buckets.{scope}.requests.create',
'buckets.{scope}.requests.read',
'buckets.{scope}.requests.update',
'buckets.{scope}.requests.delete',
'files.{scope}.requests.create',
'files.{scope}.requests.read',
'files.{scope}.requests.update',
'files.{scope}.requests.delete',
'buckets.{scope}.count.total',
'files.{scope}.count.total',
'files.{scope}.storage.size'
];
foreach ($storageMertics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value !== 0) {
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->count($metric . $storageTags, $value);
}
}
$sessionsMetrics = [
'sessions.{scope}.requests.create',
'sessions.{scope}.requests.update',
'sessions.{scope}.requests.delete',
];
foreach ($sessionsMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? '');
$this->statsd->count($metric . $sessionTags, $value);
}
}
$functionId = $this->params['functionId'] ?? '';
$functionExecution = $this->params['executions.{scope}.compute'] ?? 0;
$functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms
$functionExecutionStatus = $this->params['executionStatus'] ?? '';
$functionBuild = $this->params['builds.{scope}.compute'] ?? 0;
$functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
$functionBuildStatus = $this->params['buildStatus'] ?? '';
$functionCompute = $functionExecutionTime + $functionBuildTime;
$functionTags = $tags . ',functionId=' . $functionId;
$deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0;
$storageSize = $this->params['files.{scope}.storage.size'] ?? 0;
if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) {
$this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize);
}
if ($deploymentSize !== 0) {
$this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize);
}
if ($functionExecution >= 1) {
$this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus);
if ($functionExecutionTime > 0) {
$this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime);
}
}
if ($functionBuild >= 1) {
$this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
$this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
}
if ($functionBuild + $functionExecution >= 1) {
$this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);
}
$this->reset();
}
public function reset(): self
{
$this->params = [];
$this->namespace = 'appwrite.usage';
return $this;
}
}

2
src/Appwrite/Utopia/Database/Validator/Queries/Base.php
View file

@ -25,7 +25,7 @@ class Base extends Queries
public function __construct(string $collection, array $allowedAttributes)
{
$config = Config::getParam('collections', []);
$collections = array_merge($config['console'], $config['projects'], $config['buckets'], $config['databases']);
$collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']);
$collection = $collections[$collection];
// array for constant lookup time
$allowedAttributesLookup = [];

3
src/Appwrite/Utopia/Database/Validator/Queries/Teams.php
View file

@ -6,7 +6,8 @@ class Teams extends Base
{
public const ALLOWED_ATTRIBUTES = [
'name',
'total'
'total',
'billingPlan'
];
/**

6
src/Appwrite/Utopia/Response.php
View file

@ -3,7 +3,6 @@
namespace Appwrite\Utopia;
use Exception;
use Swoole\Http\Request as SwooleRequest;
use Utopia\Swoole\Response as SwooleResponse;
use Swoole\Http\Response as SwooleHTTPResponse;
use Utopia\Database\Document;
@ -39,7 +38,6 @@ use Appwrite\Utopia\Response\Model\Continent;
use Appwrite\Utopia\Response\Model\Country;
use Appwrite\Utopia\Response\Model\Currency;
use Appwrite\Utopia\Response\Model\Document as ModelDocument;
use Appwrite\Utopia\Response\Model\Domain;
use Appwrite\Utopia\Response\Model\Error;
use Appwrite\Utopia\Response\Model\ErrorDev;
use Appwrite\Utopia\Response\Model\Execution;
@ -60,7 +58,6 @@ use Appwrite\Utopia\Response\Model\Locale;
use Appwrite\Utopia\Response\Model\Log;
use Appwrite\Utopia\Response\Model\Membership;
use Appwrite\Utopia\Response\Model\Metric;
use Appwrite\Utopia\Response\Model\Permissions;
use Appwrite\Utopia\Response\Model\Phone;
use Appwrite\Utopia\Response\Model\Platform;
use Appwrite\Utopia\Response\Model\Project;
@ -79,6 +76,7 @@ use Appwrite\Utopia\Response\Model\HealthTime;
use Appwrite\Utopia\Response\Model\HealthVersion;
use Appwrite\Utopia\Response\Model\Installation;
use Appwrite\Utopia\Response\Model\LocaleCode;
use Appwrite\Utopia\Response\Model\MetricBreakdown;
use Appwrite\Utopia\Response\Model\Provider;
use Appwrite\Utopia\Response\Model\ProviderRepository;
use Appwrite\Utopia\Response\Model\Runtime;
@ -113,6 +111,7 @@ class Response extends SwooleResponse
public const MODEL_ERROR = 'error';
public const MODEL_METRIC = 'metric';
public const MODEL_METRIC_LIST = 'metricList';
public const MODEL_METRIC_BREAKDOWN = 'metricBreakdown';
public const MODEL_ERROR_DEV = 'errorDev';
public const MODEL_BASE_LIST = 'baseList';
public const MODEL_USAGE_DATABASES = 'usageDatabases';
@ -394,6 +393,7 @@ class Response extends SwooleResponse
->setModel(new HealthTime())
->setModel(new HealthVersion())
->setModel(new Metric())
->setModel(new MetricBreakdown())
->setModel(new UsageDatabases())
->setModel(new UsageDatabase())
->setModel(new UsageCollection())

52
src/Appwrite/Utopia/Response/Model/MetricBreakdown.php Normal file
View file

@ -0,0 +1,52 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class MetricBreakdown extends Model
{
public function __construct()
{
$this
->addRule('resourceId', [
'type' => self::TYPE_STRING,
'description' => 'Resource ID.',
'default' => '',
'example' => '5e5ea5c16897e',
])
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Resource name.',
'default' => '',
'example' => 'Documents',
])
->addRule('value', [
'type' => self::TYPE_INTEGER,
'description' => 'The value of this metric at the timestamp.',
'default' => 0,
'example' => 1,
]);
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Metric Breakdown';
}
/**
* Get Collection
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_METRIC_BREAKDOWN;
}
}

50
src/Appwrite/Utopia/Response/Model/UsageBuckets.php
View file

@ -12,48 +12,32 @@ class UsageBuckets extends Model
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('filesCount', [
->addRule('filesTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of bucket files.',
'default' => 0,
'example' => 0,
])
->addRule('filesStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of bucket files storage (in bytes).',
'default' => 0,
'example' => 0,
])
->addRule('files', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files in this bucket.',
'description' => 'Aggregated number of bucket files per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesStorage', [
->addRule('storage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total storage of files in this bucket.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'description' => 'Aggregated number of bucket storage files (in bytes) per period.',
'default' => [],
'example' => [],
'array' => true

38
src/Appwrite/Utopia/Response/Model/UsageCollection.php
View file

@ -12,41 +12,19 @@ class UsageCollection extends Model
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
->addRule('documentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of of documents.',
'default' => 0,
'example' => 0,
])
->addRule('documentsCreate', [
->addRule('documents', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'description' => 'Aggregated number of documents per period.',
'default' => [],
'example' => [],
'array' => true

78
src/Appwrite/Utopia/Response/Model/UsageDatabase.php
View file

@ -12,76 +12,32 @@ class UsageDatabase extends Model
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
->addRule('collectionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of collections.',
'default' => 0,
'example' => 0,
])
->addRule('documentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of documents.',
'default' => 0,
'example' => 0,
])
->addRule('collections', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'description' => 'Aggregated number of collections per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
->addRule('documents', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'description' => 'Aggregated number of documents per period.',
'default' => [],
'example' => [],
'array' => true

116
src/Appwrite/Utopia/Response/Model/UsageDatabases.php
View file

@ -12,111 +12,45 @@ class UsageDatabases extends Model
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('databasesCount', [
->addRule('databasesTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of databases.',
'default' => 0,
'example' => 0,
])
->addRule('collectionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of collections.',
'default' => 0,
'example' => 0,
])
->addRule('documentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of documents.',
'default' => 0,
'example' => 0,
])
->addRule('databases', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'description' => 'Aggregated number of databases per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCount', [
->addRule('collections', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'description' => 'Aggregated number of collections per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
->addRule('documents', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'description' => 'Aggregated number of documents per period.',
'default' => [],
'example' => [],
'array' => true

100
src/Appwrite/Utopia/Response/Model/UsageFunction.php
View file

@ -16,58 +16,94 @@ class UsageFunction extends Model
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
->addRule('deploymentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of function deployments.',
'default' => 0,
'example' => 0,
])
->addRule('executionsFailure', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'default' => [],
'example' => [],
'array' => true
->addRule('deploymentsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of function deployments storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of function builds.',
'default' => 0,
'example' => 0,
])
->addRule('buildsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'total aggregated sum of function builds storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTimeTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of function builds compute time.',
'default' => 0,
'example' => 0,
])
->addRule('executionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of function executions.',
'default' => 0,
'example' => 0,
])
->addRule('executionsTimeTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of function executions compute time.',
'default' => 0,
'example' => 0,
])
->addRule('deployments', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function builds.',
'description' => 'Aggregated number of function deployments per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'description' => 'Aggregated number of function deployments storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
->addRule('builds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated number of function builds per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of function builds storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated sum of function builds compute time per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of function executions per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of function executions compute time per period.',
'default' => [],
'example' => [],
'array' => true

113
src/Appwrite/Utopia/Response/Model/UsageFunctions.php
View file

@ -12,62 +12,111 @@ class UsageFunctions extends Model
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
->addRule('functionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of functions.',
'default' => 0,
'example' => 0,
])
->addRule('executionsFailure', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => [],
'array' => true
->addRule('deploymentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of functions deployments.',
'default' => 0,
'example' => 0,
])
->addRule('executionsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'default' => [],
'example' => [],
'array' => true
->addRule('deploymentsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of functions deployment storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of functions build.',
'default' => 0,
'example' => 0,
])
->addRule('buildsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'total aggregated sum of functions build storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTimeTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of functions build compute time.',
'default' => 0,
'example' => 0,
])
->addRule('executionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of functions execution.',
'default' => 0,
'example' => 0,
])
->addRule('executionsTimeTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of functions execution compute time.',
'default' => 0,
'example' => 0,
])
->addRule('functions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function builds.',
'description' => 'Aggregated number of functions per period.',
'default' => 0,
'example' => 0,
'array' => true
])
->addRule('deployments', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of functions deployment per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'description' => 'Aggregated number of functions deployment storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
->addRule('builds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated number of functions build per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of functions build storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated sum of functions build compute time per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of functions execution per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of functions execution compute time per period.',
'default' => [],
'example' => [],
'array' => true

84
src/Appwrite/Utopia/Response/Model/UsageProject.php
View file

@ -10,64 +10,80 @@ class UsageProject extends Model
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
->addRule('executionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of function executions.',
'default' => 0,
'example' => 0,
])
->addRule('documentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of documents.',
'default' => 0,
'example' => 0,
])
->addRule('databasesTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of databases.',
'default' => 0,
'example' => 0,
])
->addRule('usersTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of users.',
'default' => 0,
'example' => 0,
])
->addRule('filesStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of files storage size (in bytes).',
'default' => 0,
'example' => 0,
])
->addRule('bucketsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of buckets.',
'default' => 0,
'example' => 0,
])
->addRule('requests', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of requests.',
'description' => 'Aggregated number of requests per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('network', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for consumed bandwidth.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documents', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databases', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of databases.',
'description' => 'Aggregated number of consumed bandwidth per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('users', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of users.',
'description' => 'Aggregated number of users per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('storage', [
->addRule('executions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'description' => 'Aggregated number of executions per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buckets', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of buckets.',
->addRule('executionsBreakdown', [
'type' => Response::MODEL_METRIC_BREAKDOWN,
'description' => 'Aggregated breakdown in totals of executions by functions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsBreakdown', [
'type' => Response::MODEL_METRIC_BREAKDOWN,
'description' => 'Aggregated breakdown in totals of usage by buckets.',
'default' => [],
'example' => [],
'array' => true

106
src/Appwrite/Utopia/Response/Model/UsageStorage.php
View file

@ -12,83 +12,45 @@ class UsageStorage extends Model
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('bucketsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of buckets',
'default' => 0,
'example' => 0,
])
->addRule('filesTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of files.',
'default' => 0,
'example' => 0,
])
->addRule('filesStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of files storage (in bytes).',
'default' => 0,
'example' => 0,
])
->addRule('buckets', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of buckets per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('files', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of files per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('storage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of buckets.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'description' => 'Aggregated number of files storage (in bytes) per period .',
'default' => [],
'example' => [],
'array' => true

66
src/Appwrite/Utopia/Response/Model/UsageUsers.php
View file

@ -12,62 +12,34 @@ class UsageUsers extends Model
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('usersCount', [
->addRule('usersTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of statistics of users.',
'default' => 0,
'example' => 0,
])
->addRule('sessionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of active sessions.',
'default' => 0,
'example' => 0,
])
->addRule('users', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of users.',
'description' => 'Aggregated number of users per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersCreate', [
->addRule('sessions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsProviderCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions deleted.',
'description' => 'Aggregated number of active sessions per period.',
'default' => [],
'example' => [],
'array' => true

46
tests/e2e/General/HTTPTest.php
View file

@ -171,4 +171,50 @@ class HTTPTest extends Scope
$this->assertEquals(200, $response['headers']['status-code']);
}
public function testCors()
{
/**
* Test for SUCCESS
*/
$endpoint = '/v1/projects'; // Can be any non-404 route
$response = $this->client->call(Client::METHOD_GET, $endpoint);
$this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
$response = $this->client->call(Client::METHOD_GET, $endpoint, [
'origin' => 'http://localhost',
]);
$this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
$response = $this->client->call(Client::METHOD_GET, $endpoint, [
'origin' => 'http://appwrite.io',
]);
$this->assertEquals('http://appwrite.io', $response['headers']['access-control-allow-origin']);
$response = $this->client->call(Client::METHOD_GET, $endpoint, [
'origin' => 'https://appwrite.io',
]);
$this->assertEquals('https://appwrite.io', $response['headers']['access-control-allow-origin']);
$response = $this->client->call(Client::METHOD_GET, $endpoint, [
'origin' => 'http://cloud.appwrite.io',
]);
$this->assertEquals('http://cloud.appwrite.io', $response['headers']['access-control-allow-origin']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, $endpoint, [
'origin' => 'http://google.com',
]);
$this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
}
}

1543
tests/e2e/General/UsageTest.php
View file

File diff suppressed because it is too large Load diff

42
tests/e2e/Services/Account/AccountBase.php
View file

@ -1554,4 +1554,46 @@ trait AccountBase
return $data;
}
public function testDeleteAccount(): void
{
$email = uniqid() . 'user@localhost.test';
$password = 'password';
$name = 'User Name';
$response = $this->client->call(Client::METHOD_POST, '/account', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]), [
'userId' => ID::unique(),
'email' => $email,
'password' => $password,
'name' => $name,
]);
$this->assertEquals($response['headers']['status-code'], 201);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]), [
'email' => $email,
'password' => $password,
]);
$this->assertEquals($response['headers']['status-code'], 201);
$session = $response['cookies']['a_session_' . $this->getProject()['$id']];
$response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]));
$this->assertEquals($response['headers']['status-code'], 204);
}
}

52
tests/e2e/Services/Databases/DatabasesConsoleClientTest.php
View file

@ -193,6 +193,46 @@ class DatabasesConsoleClientTest extends Scope
$this->assertEquals($response['body'], "");
}
/**
* @depends testCreateCollection
*/
public function testGetDatabaseUsage(array $data)
{
$databaseId = $data['databaseId'];
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
$this->assertEquals(400, $response['headers']['status-code']);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(5, count($response['body']));
$this->assertEquals('24h', $response['body']['range']);
$this->assertIsNumeric($response['body']['documentsTotal']);
$this->assertIsNumeric($response['body']['collectionsTotal']);
$this->assertIsArray($response['body']['collections']);
$this->assertIsArray($response['body']['documents']);
}
/**
* @depends testCreateCollection
*/
@ -230,15 +270,11 @@ class DatabasesConsoleClientTest extends Scope
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 6);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documentsCount']);
$this->assertIsArray($response['body']['documentsCreate']);
$this->assertIsArray($response['body']['documentsRead']);
$this->assertIsArray($response['body']['documentsUpdate']);
$this->assertIsArray($response['body']['documentsDelete']);
$this->assertEquals(3, count($response['body']));
$this->assertEquals('24h', $response['body']['range']);
$this->assertIsNumeric($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['documents']);
}
/**

27
tests/e2e/Services/Functions/FunctionsConsoleClientTest.php
View file

@ -91,17 +91,24 @@ class FunctionsConsoleClientTest extends Scope
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['executionsTotal']);
$this->assertIsArray($response['body']['executionsFailure']);
$this->assertIsArray($response['body']['executionsSuccess']);
$this->assertIsArray($response['body']['executionsTime']);
$this->assertIsArray($response['body']['buildsTotal']);
$this->assertIsArray($response['body']['buildsFailure']);
$this->assertIsArray($response['body']['buildsSuccess']);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(15, count($response['body']));
$this->assertEquals('24h', $response['body']['range']);
$this->assertIsNumeric($response['body']['deploymentsTotal']);
$this->assertIsNumeric($response['body']['deploymentsStorageTotal']);
$this->assertIsNumeric($response['body']['buildsTotal']);
$this->assertIsNumeric($response['body']['buildsStorageTotal']);
$this->assertIsNumeric($response['body']['buildsTimeTotal']);
$this->assertIsNumeric($response['body']['executionsTotal']);
$this->assertIsNumeric($response['body']['executionsTimeTotal']);
$this->assertIsArray($response['body']['deployments']);
$this->assertIsArray($response['body']['deploymentsStorage']);
$this->assertIsArray($response['body']['builds']);
$this->assertIsArray($response['body']['buildsTime']);
$this->assertIsArray($response['body']['buildsStorage']);
$this->assertIsArray($response['body']['buildsTime']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['executionsTime']);
}
/**

41
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
View file

@ -8,6 +8,7 @@ use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectConsole;
use Tests\E2E\Scopes\SideClient;
use Tests\E2E\Client;
use Tests\E2E\General\UsageTest;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\ID;
@ -440,33 +441,36 @@ class ProjectsConsoleClientTest extends Scope
*/
public function testGetProjectUsage($data): array
{
$id = $data['projectId'] ?? '';
$this->markTestIncomplete(
'This test is failing right now due to functions collection.'
);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/projects/' . $id . '/usage', array_merge([
$response = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
], $this->getHeaders()), [
'startDate' => UsageTest::getToday(),
'endDate' => UsageTest::getTomorrow(),
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals(8, count($response['body']));
$this->assertNotEmpty($response['body']);
$this->assertEquals('30d', $response['body']['range']);
$this->assertIsArray($response['body']['requests']);
$this->assertIsArray($response['body']['network']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['documents']);
$this->assertIsArray($response['body']['databases']);
$this->assertIsArray($response['body']['buckets']);
$this->assertIsArray($response['body']['users']);
$this->assertIsArray($response['body']['storage']);
$this->assertIsNumeric($response['body']['executionsTotal']);
$this->assertIsNumeric($response['body']['documentsTotal']);
$this->assertIsNumeric($response['body']['databasesTotal']);
$this->assertIsNumeric($response['body']['bucketsTotal']);
$this->assertIsNumeric($response['body']['usersTotal']);
$this->assertIsNumeric($response['body']['filesStorageTotal']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/projects/empty', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@ -485,7 +489,7 @@ class ProjectsConsoleClientTest extends Scope
}
/**
* @depends testGetProjectUsage
* @depends testCreateProject
*/
public function testUpdateProject($data): array
{
@ -669,7 +673,7 @@ class ProjectsConsoleClientTest extends Scope
return $data;
}
/** @depends testGetProjectUsage */
/** @depends testCreateProject */
public function testUpdateProjectAuthDuration($data): array
{
$id = $data['projectId'];
@ -789,7 +793,7 @@ class ProjectsConsoleClientTest extends Scope
}
/**
* @depends testGetProjectUsage
* @depends testCreateProject
*/
public function testUpdateProjectOAuth($data): array
{
@ -900,7 +904,7 @@ class ProjectsConsoleClientTest extends Scope
}
/**
* @depends testGetProjectUsage
* @depends testCreateProject
*/
public function testUpdateProjectAuthStatus($data): array
{
@ -1045,7 +1049,7 @@ class ProjectsConsoleClientTest extends Scope
}
/**
* @depends testGetProjectUsage
* @depends testCreateProject
*/
public function testUpdateProjectAuthLimit($data): array
{
@ -1661,7 +1665,6 @@ class ProjectsConsoleClientTest extends Scope
foreach ($response['body'] as $key => $value) {
if (\preg_match($pattern, $key)) {
\var_dump('Matched key: ' . $key);
$matches[$key] = $value;
}
}

32
tests/e2e/Services/Storage/StorageConsoleClientTest.php
View file

@ -38,11 +38,15 @@ class StorageConsoleClientTest extends Scope
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(12, count($response['body']));
$this->assertEquals($response['body']['range'], '24h');
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(7, count($response['body']));
$this->assertEquals('24h', $response['body']['range']);
$this->assertIsNumeric($response['body']['bucketsTotal']);
$this->assertIsNumeric($response['body']['filesTotal']);
$this->assertIsNumeric($response['body']['filesStorageTotal']);
$this->assertIsArray($response['body']['buckets']);
$this->assertIsArray($response['body']['files']);
$this->assertIsArray($response['body']['storage']);
$this->assertIsArray($response['body']['filesCount']);
}
public function testGetStorageBucketUsage()
@ -70,7 +74,7 @@ class StorageConsoleClientTest extends Scope
'range' => '32h'
]);
$this->assertEquals($response['headers']['status-code'], 400);
$this->assertEquals(400, $response['headers']['status-code']);
// TODO: Uncomment once we implement check for missing bucketId in the usage endpoint.
@ -81,7 +85,7 @@ class StorageConsoleClientTest extends Scope
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 404);
$this->assertEquals(404, $response['headers']['status-code']);
/**
* Test for SUCCESS
@ -93,14 +97,12 @@ class StorageConsoleClientTest extends Scope
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 7);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['filesCount']);
$this->assertIsArray($response['body']['filesCreate']);
$this->assertIsArray($response['body']['filesRead']);
$this->assertIsArray($response['body']['filesUpdate']);
$this->assertIsArray($response['body']['filesDelete']);
$this->assertIsArray($response['body']['filesStorage']);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(5, count($response['body']));
$this->assertEquals('24h', $response['body']['range']);
$this->assertIsNumeric($response['body']['filesTotal']);
$this->assertIsNumeric($response['body']['filesStorageTotal']);
$this->assertIsArray($response['body']['files']);
$this->assertIsArray($response['body']['storage']);
}
}

49
tests/e2e/Services/Users/UsersConsoleClientTest.php
View file

@ -23,17 +23,6 @@ class UsersConsoleClientTest extends Scope
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 400);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'some-random-provider'
]);
$this->assertEquals($response['headers']['status-code'], 400);
@ -46,38 +35,14 @@ class UsersConsoleClientTest extends Scope
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['usersCount']);
$this->assertIsArray($response['body']['usersCreate']);
$this->assertIsArray($response['body']['usersRead']);
$this->assertIsArray($response['body']['usersUpdate']);
$this->assertIsArray($response['body']['usersDelete']);
$this->assertIsArray($response['body']['sessionsCreate']);
$this->assertIsArray($response['body']['sessionsProviderCreate']);
$this->assertIsArray($response['body']['sessionsDelete']);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['usersCount']);
$this->assertIsArray($response['body']['usersCreate']);
$this->assertIsArray($response['body']['usersRead']);
$this->assertIsArray($response['body']['usersUpdate']);
$this->assertIsArray($response['body']['usersDelete']);
$this->assertIsArray($response['body']['sessionsCreate']);
$this->assertIsArray($response['body']['sessionsProviderCreate']);
$this->assertIsArray($response['body']['sessionsDelete']);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(5, count($response['body']));
$this->assertEquals('24h', $response['body']['range']);
$this->assertIsNumeric($response['body']['usersTotal']);
$this->assertIsNumeric($response['body']['sessionsTotal']);
$this->assertIsArray($response['body']['users']);
$this->assertIsArray($response['body']['sessions']);
}
}

79
tests/unit/Usage/StatsTest.php
View file

@ -2,70 +2,53 @@
namespace Tests\Unit\Usage;
use Appwrite\Usage\Stats;
use Appwrite\URL\URL as AppwriteURL;
use PHPUnit\Framework\TestCase;
use Utopia\App;
use Utopia\DSN\DSN;
use Utopia\Queue;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class StatsTest extends TestCase
{
/**
* @var Stats
*/
protected $object = null;
protected ?Connection $connection = null;
protected ?Client $client = null;
protected const QUEUE_NAME = 'usage-test-q';
public function setUp(): void
{
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$env = App::getEnv('_APP_CONNECTIONS_QUEUE', AppwriteURL::unparse([
'scheme' => 'redis',
'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
'user' => App::getEnv('_APP_REDIS_USER', ''),
'pass' => App::getEnv('_APP_REDIS_PASS', ''),
]));
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
$this->object = new Stats($statsd);
$dsn = explode('=', $env);
$dsn = count($dsn) > 1 ? $dsn[1] : $dsn[0];
$dsn = new DSN($dsn);
$this->connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort());
$this->client = new Client(self::QUEUE_NAME, $this->connection);
}
public function tearDown(): void
{
}
public function testNamespace(): void
public function testSamePayload(): void
{
$this->object->setNamespace('appwritetest.usage');
$this->assertEquals('appwritetest.usage', $this->object->getNamespace());
}
$inToQueue = [
'key_1' => 'value_1',
'key_2' => 'value_2',
];
public function testParams(): void
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('projectInternalId', 1)
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(1, $this->object->getParam('projectInternalId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->submit();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
}
public function testReset(): void
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->reset();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
$this->assertEquals('appwrite.usage', $this->object->getNamespace());
$result = $this->client->enqueue($inToQueue);
$this->assertTrue($result);
$outFromQueue = $this->connection->leftPopArray('utopia-queue.queue.' . self::QUEUE_NAME, 0)['payload'];
$this->assertNotEmpty($outFromQueue);
$this->assertSame($inToQueue, $outFromQueue);
}
}