Merge branch 'refactor-usage-sn' of github.com:appwrite/appwrite into refactor-hamster-usage

 Conflicts:
	src/Appwrite/Platform/Tasks/CalcTierStats.php
This commit is contained in:
shimon 2023-12-21 14:25:06 +02:00
commit 4e24e3cdc2
22 changed files with 325 additions and 280 deletions

2
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "app/console"] [submodule "app/console"]
path = app/console path = app/console
url = https://github.com/appwrite/console url = https://github.com/appwrite/console
branch = billing branch = 3.3.4

View file

@ -18,6 +18,63 @@ $auth = Config::getParam('auth', []);
*/ */
$commonCollections = [ $commonCollections = [
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'resource',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_accessedAt',
'type' => Database::INDEX_KEY,
'attributes' => ['accessedAt'],
'lengths' => [],
'orders' => [],
],
[
'$id' => '_key_resource',
'type' => Database::INDEX_KEY,
'attributes' => ['resource'],
'lengths' => [],
'orders' => [],
],
],
],
'users' => [ 'users' => [
'$collection' => ID::custom(Database::METADATA), '$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('users'), '$id' => ID::custom('users'),
@ -1270,10 +1327,10 @@ $commonCollections = [
] ]
], ],
'stats' => [ 'stats_v2' => [
'$collection' => ID::custom(Database::METADATA), '$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('stats'), '$id' => ID::custom('stats_v2'),
'name' => 'Stats', 'name' => 'stats_v2',
'attributes' => [ 'attributes' => [
[ [
'$id' => ID::custom('metric'), '$id' => ID::custom('metric'),
@ -2872,63 +2929,6 @@ $projectCollections = array_merge([
], ],
], ],
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'resource',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_accessedAt',
'type' => Database::INDEX_KEY,
'attributes' => ['accessedAt'],
'lengths' => [],
'orders' => [],
],
[
'$id' => '_key_resource',
'type' => Database::INDEX_KEY,
'attributes' => ['resource'],
'lengths' => [],
'orders' => [],
],
],
],
'variables' => [ 'variables' => [
'$collection' => Database::METADATA, '$collection' => Database::METADATA,
'$id' => 'variables', '$id' => 'variables',

View file

@ -1,15 +1,9 @@
<p>{{hello}}</p> <p>{{hello}},</p>
<br>
<p>{{body}}</p> <p>{{body}}</p>
<p><a href="{{redirect}}" target="_blank">{{redirect}}</a></p>
<a href="{{redirect}}" target="_blank">{{redirect}}</a>
<p>{{footer}}</p> <p>{{footer}}</p>
<p style="margin-bottom: 32px">
<br> {{thanks}},
<br/>
<p>{{thanks}}</p> {{signature}}
</p>
<p>{{signature}}</p>

@ -1 +1 @@
Subproject commit 3b6dd23ff5bf0633508ae7ef2c09640e3da4468b Subproject commit b3a0348b5516c969506370726c784c0fe1f3c62a

View file

@ -45,6 +45,7 @@ use Utopia\Validator\WhiteList;
use Appwrite\Auth\Validator\PasswordHistory; use Appwrite\Auth\Validator\PasswordHistory;
use Appwrite\Auth\Validator\PasswordDictionary; use Appwrite\Auth\Validator\PasswordDictionary;
use Appwrite\Auth\Validator\PersonalData; use Appwrite\Auth\Validator\PersonalData;
use Appwrite\Hooks\Hooks;
$oauthDefaultSuccess = '/auth/oauth2/success'; $oauthDefaultSuccess = '/auth/oauth2/success';
$oauthDefaultFailure = '/auth/oauth2/failure'; $oauthDefaultFailure = '/auth/oauth2/failure';
@ -76,7 +77,8 @@ App::post('/v1/account')
->inject('project') ->inject('project')
->inject('dbForProject') ->inject('dbForProject')
->inject('queueForEvents') ->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) { ->inject('hooks')
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$email = \strtolower($email); $email = \strtolower($email);
if ('console' === $project->getId()) { if ('console' === $project->getId()) {
@ -117,6 +119,8 @@ App::post('/v1/account')
} }
} }
$hooks->trigger('passwordValidator', [$project, $password, $user]);
$passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
$password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS); $password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
try { try {

View file

@ -3563,7 +3563,7 @@ App::get('/v1/databases/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -3571,7 +3571,7 @@ App::get('/v1/databases/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),
@ -3647,7 +3647,7 @@ App::get('/v1/databases/:databaseId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -3655,7 +3655,7 @@ App::get('/v1/databases/:databaseId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),
@ -3733,7 +3733,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -3741,7 +3741,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),

View file

@ -484,7 +484,7 @@ App::get('/v1/functions/:functionId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -492,7 +492,7 @@ App::get('/v1/functions/:functionId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),
@ -576,7 +576,7 @@ App::get('/v1/functions/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -584,7 +584,7 @@ App::get('/v1/functions/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),
@ -1730,6 +1730,13 @@ App::post('/v1/functions/:functionId/executions')
->setAttribute('responseStatusCode', 500) ->setAttribute('responseStatusCode', 500)
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode()); ->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
Console::error($th->getMessage()); Console::error($th->getMessage());
} finally {
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
;
} }
if ($function->getAttribute('logging')) { if ($function->getAttribute('logging')) {
@ -1737,15 +1744,6 @@ App::post('/v1/functions/:functionId/executions')
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
} }
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function
;
$roles = Authorization::getRoles(); $roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles); $isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles); $isAppUser = Auth::isAppUser($roles);

View file

@ -73,7 +73,7 @@ App::get('/v1/project/usage')
Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) { Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
foreach ($metrics['total'] as $metric) { foreach ($metrics['total'] as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -81,7 +81,7 @@ App::get('/v1/project/usage')
} }
foreach ($metrics['period'] as $metric) { foreach ($metrics['period'] as $metric) {
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::greaterThanEqual('time', $firstDay), Query::greaterThanEqual('time', $firstDay),
@ -116,7 +116,7 @@ App::get('/v1/project/usage')
$id = $function->getId(); $id = $function->getId();
$name = $function->getAttribute('name'); $name = $function->getAttribute('name');
$metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS); $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
$value = $dbForProject->findOne('stats', [ $value = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -132,7 +132,7 @@ App::get('/v1/project/usage')
$id = $bucket->getId(); $id = $bucket->getId();
$name = $bucket->getAttribute('name'); $name = $bucket->getAttribute('name');
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
$value = $dbForProject->findOne('stats', [ $value = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);

View file

@ -1491,7 +1491,7 @@ App::get('/v1/storage/usage')
$total = []; $total = [];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -1499,7 +1499,7 @@ App::get('/v1/storage/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),
@ -1576,7 +1576,7 @@ App::get('/v1/storage/:bucketId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
foreach ($metrics as $metric) { foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -1584,7 +1584,7 @@ App::get('/v1/storage/:bucketId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),

View file

@ -1238,7 +1238,7 @@ App::get('/v1/users/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $count => $metric) { foreach ($metrics as $count => $metric) {
$result = $dbForProject->findOne('stats', [ $result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', ['inf']) Query::equal('period', ['inf'])
]); ]);
@ -1246,7 +1246,7 @@ App::get('/v1/users/usage')
$stats[$metric]['total'] = $result['value'] ?? 0; $stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit']; $limit = $days['limit'];
$period = $days['period']; $period = $days['period'];
$results = $dbForProject->find('stats', [ $results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),

View file

@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException;
use Appwrite\Event\Build; use Appwrite\Event\Build;
use Appwrite\Event\Certificate; use Appwrite\Event\Certificate;
use Appwrite\Event\Func; use Appwrite\Event\Func;
use Appwrite\Hooks\Hooks;
use MaxMind\Db\Reader; use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer; use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy; use Swoole\Database\PDOProxy;
@ -828,6 +829,9 @@ $register->set('passwordsDictionary', function () {
$register->set('promiseAdapter', function () { $register->set('promiseAdapter', function () {
return new Swoole(); return new Swoole();
}); });
$register->set('hooks', function () {
return new Hooks();
});
/* /*
* Localization * Localization
*/ */
@ -867,6 +871,10 @@ App::setResource('logger', function ($register) {
return $register->get('logger'); return $register->get('logger');
}, ['register']); }, ['register']);
App::setResource('hooks', function ($register) {
return $register->get('hooks');
}, ['register']);
App::setResource('loggerBreadcrumbs', function () { App::setResource('loggerBreadcrumbs', function () {
return []; return [];
}); });

View file

@ -72,6 +72,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
return $adapter; return $adapter;
}, ['cache', 'register', 'message', 'dbForConsole']); }, ['cache', 'register', 'message', 'dbForConsole']);
Server::setResource('project', function (Message $message, Database $dbForConsole) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
if ($project->getId() === 'console') {
return $project;
}
return $dbForConsole->getDocument('projects', $project->getId());
;
}, ['message', 'dbForConsole']);
Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
$databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
@ -103,22 +114,16 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
}; };
}, ['pools', 'dbForConsole', 'cache']); }, ['pools', 'dbForConsole', 'cache']);
Server::setResource('getProjectAbuseRetention', function () { Server::setResource('abuseRetention', function () {
return function (Document $project) { return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
};
}); });
Server::setResource('getProjectAuditRetention', function () { Server::setResource('auditRetention', function () {
return function (Document $project) { return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
};
}); });
Server::setResource('getProjectExecutionRetention', function () { Server::setResource('executionRetention', function () {
return function (Document $project) { return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
};
}); });
Server::setResource('cache', function (Registry $register) { Server::setResource('cache', function (Registry $register) {

82
composer.lock generated
View file

@ -2768,16 +2768,16 @@
}, },
{ {
"name": "nikic/php-parser", "name": "nikic/php-parser",
"version": "v4.17.1", "version": "v4.18.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/nikic/PHP-Parser.git", "url": "https://github.com/nikic/PHP-Parser.git",
"reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d" "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999",
"reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -2818,9 +2818,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/nikic/PHP-Parser/issues", "issues": "https://github.com/nikic/PHP-Parser/issues",
"source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1" "source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0"
}, },
"time": "2023-08-13T19:53:39+00:00" "time": "2023-12-10T21:03:43+00:00"
}, },
{ {
"name": "phar-io/manifest", "name": "phar-io/manifest",
@ -3103,29 +3103,29 @@
}, },
{ {
"name": "phpspec/prophecy", "name": "phpspec/prophecy",
"version": "v1.17.0", "version": "v1.18.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/phpspec/prophecy.git", "url": "https://github.com/phpspec/prophecy.git",
"reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2" "reference": "d4f454f7e1193933f04e6500de3e79191648ed0c"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/phpspec/prophecy/zipball/15873c65b207b07765dbc3c95d20fdf4a320cbe2", "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d4f454f7e1193933f04e6500de3e79191648ed0c",
"reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2", "reference": "d4f454f7e1193933f04e6500de3e79191648ed0c",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"doctrine/instantiator": "^1.2 || ^2.0", "doctrine/instantiator": "^1.2 || ^2.0",
"php": "^7.2 || 8.0.* || 8.1.* || 8.2.*", "php": "^7.2 || 8.0.* || 8.1.* || 8.2.* || 8.3.*",
"phpdocumentor/reflection-docblock": "^5.2", "phpdocumentor/reflection-docblock": "^5.2",
"sebastian/comparator": "^3.0 || ^4.0", "sebastian/comparator": "^3.0 || ^4.0 || ^5.0",
"sebastian/recursion-context": "^3.0 || ^4.0" "sebastian/recursion-context": "^3.0 || ^4.0 || ^5.0"
}, },
"require-dev": { "require-dev": {
"phpspec/phpspec": "^6.0 || ^7.0", "phpspec/phpspec": "^6.0 || ^7.0",
"phpstan/phpstan": "^1.9", "phpstan/phpstan": "^1.9",
"phpunit/phpunit": "^8.0 || ^9.0" "phpunit/phpunit": "^8.0 || ^9.0 || ^10.0"
}, },
"type": "library", "type": "library",
"extra": { "extra": {
@ -3158,6 +3158,7 @@
"keywords": [ "keywords": [
"Double", "Double",
"Dummy", "Dummy",
"dev",
"fake", "fake",
"mock", "mock",
"spy", "spy",
@ -3165,9 +3166,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/phpspec/prophecy/issues", "issues": "https://github.com/phpspec/prophecy/issues",
"source": "https://github.com/phpspec/prophecy/tree/v1.17.0" "source": "https://github.com/phpspec/prophecy/tree/v1.18.0"
}, },
"time": "2023-02-02T15:41:36+00:00" "time": "2023-12-07T16:22:33+00:00"
}, },
{ {
"name": "phpstan/phpdoc-parser", "name": "phpstan/phpdoc-parser",
@ -4654,16 +4655,16 @@
}, },
{ {
"name": "squizlabs/php_codesniffer", "name": "squizlabs/php_codesniffer",
"version": "3.7.2", "version": "3.8.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/squizlabs/PHP_CodeSniffer.git", "url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git",
"reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879" "reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/ed8e00df0a83aa96acf703f8c2979ff33341f879", "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
"reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879", "reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -4673,7 +4674,7 @@
"php": ">=5.4.0" "php": ">=5.4.0"
}, },
"require-dev": { "require-dev": {
"phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0" "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.0"
}, },
"bin": [ "bin": [
"bin/phpcs", "bin/phpcs",
@ -4692,22 +4693,45 @@
"authors": [ "authors": [
{ {
"name": "Greg Sherwood", "name": "Greg Sherwood",
"role": "lead" "role": "Former lead"
},
{
"name": "Juliette Reinders Folmer",
"role": "Current lead"
},
{
"name": "Contributors",
"homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer/graphs/contributors"
} }
], ],
"description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.", "description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.",
"homepage": "https://github.com/squizlabs/PHP_CodeSniffer", "homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
"keywords": [ "keywords": [
"phpcs", "phpcs",
"standards", "standards",
"static analysis" "static analysis"
], ],
"support": { "support": {
"issues": "https://github.com/squizlabs/PHP_CodeSniffer/issues", "issues": "https://github.com/PHPCSStandards/PHP_CodeSniffer/issues",
"source": "https://github.com/squizlabs/PHP_CodeSniffer", "security": "https://github.com/PHPCSStandards/PHP_CodeSniffer/security/policy",
"wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki" "source": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
"wiki": "https://github.com/PHPCSStandards/PHP_CodeSniffer/wiki"
}, },
"time": "2023-02-22T23:07:41+00:00" "funding": [
{
"url": "https://github.com/PHPCSStandards",
"type": "github"
},
{
"url": "https://github.com/jrfnl",
"type": "github"
},
{
"url": "https://opencollective.com/php_codesniffer",
"type": "open_collective"
}
],
"time": "2023-12-08T12:32:31+00:00"
}, },
{ {
"name": "swoole/ide-helper", "name": "swoole/ide-helper",
@ -5104,5 +5128,5 @@
"platform-overrides": { "platform-overrides": {
"php": "8.0" "php": "8.0"
}, },
"plugin-api-version": "2.2.0" "plugin-api-version": "2.6.0"
} }

View file

@ -0,0 +1,26 @@
<?php
namespace Appwrite\Hooks;
class Hooks
{
/**
* @var callable[] $hooks
*/
private static array $hooks = [];
public static function add(string $name, callable $action)
{
self::$hooks[$name] = $action;
}
/**
* @param mixed[] $params
*/
public function trigger(string $name, array $params = [])
{
if (isset(self::$hooks[$name])) {
call_user_func_array(self::$hooks[$name], $params);
}
}
}

View file

@ -91,6 +91,8 @@ class CreateInfMetric extends Action
->get($db) ->get($db)
->reclaim(); ->reclaim();
} }
Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
} }
$sum = \count($projects); $sum = \count($projects);
@ -118,11 +120,9 @@ class CreateInfMetric extends Action
try { try {
$id = \md5("_inf_{$metric}"); $id = \md5("_inf_{$metric}");
$dbForProject->deleteDocument('stats', $id); $dbForProject->deleteDocument('stats_v2', $id);
echo "_inf_{$metric} , $value \n"; $dbForProject->createDocument('stats_v2', new Document([
$dbForProject->createDocument('stats', new Document([
'$id' => $id, '$id' => $id,
'metric' => $metric, 'metric' => $metric,
'period' => 'inf', 'period' => 'inf',
@ -144,7 +144,7 @@ class CreateInfMetric extends Action
protected function getFromMetric(database $dbForProject, string $metric): int|float protected function getFromMetric(database $dbForProject, string $metric): int|float
{ {
return $dbForProject->sum('stats', 'value', [ return $dbForProject->sum('stats_v2', 'value', [
Query::equal('metric', [ Query::equal('metric', [
$metric, $metric,
]), ]),

View file

@ -240,7 +240,7 @@ class Hamster extends Action
$limit = $periodValue['limit']; $limit = $periodValue['limit'];
$period = $periodValue['period']; $period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [ $requestDocs = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]), Query::equal('metric', [$metric]),
Query::equal('period', [$period]), Query::equal('period', [$period]),
Query::limit($limit), Query::limit($limit),

View file

@ -44,18 +44,51 @@ class Maintenance extends Action
$time = DateTime::now(); $time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes); $this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$this->notifyDeleteAuditLogs($queueForDeletes); $queueForDeletes->setProject($project);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
});
$this->notifyDeleteConnections($queueForDeletes); $this->notifyDeleteConnections($queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
$this->renewCertificates($dbForConsole, $queueForCertificates); $this->renewCertificates($dbForConsole, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes); $this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
}, $interval); }, $interval);
} }
protected function foreachProject(Database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
{ {
($queueForDeletes) ($queueForDeletes)

View file

@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers; namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Executor\Executor; use Executor\Executor;
use Throwable; use Throwable;
use Utopia\Abuse\Abuse; use Utopia\Abuse\Abuse;
@ -45,17 +46,17 @@ class Deletes extends Action
->inject('getFunctionsDevice') ->inject('getFunctionsDevice')
->inject('getBuildsDevice') ->inject('getBuildsDevice')
->inject('getCacheDevice') ->inject('getCacheDevice')
->inject('getProjectAbuseRetention') ->inject('abuseRetention')
->inject('getProjectExecutionRetention') ->inject('executionRetention')
->inject('getProjectAuditRetention') ->inject('auditRetention')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $getProjectAbuseRetention, $getProjectExecutionRetention, $getProjectAuditRetention)); ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention));
} }
/** /**
* @throws Exception * @throws Exception
* @throws Throwable * @throws Throwable
*/ */
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention): void public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention): void
{ {
$payload = $message->getPayload() ?? []; $payload = $message->getPayload() ?? [];
@ -117,12 +118,12 @@ class Deletes extends Action
break; break;
case DELETE_TYPE_EXECUTIONS: case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $getProjectExecutionRetention); $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
break; break;
case DELETE_TYPE_AUDIT: case DELETE_TYPE_AUDIT:
if (!empty($datetime)) { if (!$project->isEmpty()) {
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $getProjectAuditRetention); $this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
} }
if (!$document->isEmpty()) { if (!$document->isEmpty()) {
@ -130,7 +131,7 @@ class Deletes extends Action
} }
break; break;
case DELETE_TYPE_ABUSE: case DELETE_TYPE_ABUSE:
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $getProjectAbuseRetention); $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
break; break;
case DELETE_TYPE_REALTIME: case DELETE_TYPE_REALTIME:
@ -138,10 +139,10 @@ class Deletes extends Action
break; break;
case DELETE_TYPE_SESSIONS: case DELETE_TYPE_SESSIONS:
$this->deleteExpiredSessions($dbForConsole, $getProjectDB); $this->deleteExpiredSessions($project, $getProjectDB);
break; break;
case DELETE_TYPE_USAGE: case DELETE_TYPE_USAGE:
$this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime); $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
break; break;
case DELETE_TYPE_CACHE_BY_RESOURCE: case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project, $getProjectDB, $resource); $this->deleteCacheByResource($project, $getProjectDB, $resource);
@ -340,16 +341,14 @@ class Deletes extends Action
* @return void * @return void
* @throws Exception * @throws Exception
*/ */
private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
{ {
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) { $dbForProject = $getProjectDB($project);
$dbForProject = $getProjectDB($project); // Delete Usage stats
// Delete Usage stats $this->deleteByGroup('stats_v2', [
$this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']),
Query::equal('period', ['1h']), ], $dbForProject);
], $dbForProject);
});
} }
/** /**
@ -546,16 +545,13 @@ class Deletes extends Action
* @return void * @return void
* @throws Exception * @throws Exception
*/ */
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, callable $getProjectExecutionRetention): void private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
{ {
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectExecutionRetention) { $dbForProject = $getProjectDB($project);
$dbForProject = $getProjectDB($project); // Delete Executions
$datetime = $getProjectExecutionRetention($project); $this->deleteByGroup('executions', [
// Delete Executions Query::lessThan('$createdAt', $datetime)
$this->deleteByGroup('executions', [ ], $dbForProject);
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
});
} }
/** /**
@ -564,20 +560,16 @@ class Deletes extends Action
* @return void * @return void
* @throws Exception|Throwable * @throws Exception|Throwable
*/ */
private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void private function deleteExpiredSessions(Document $project, callable $getProjectDB): void
{ {
$dbForProject = $getProjectDB($project);
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) { // Delete Sessions
$dbForProject = $getProjectDB($project); $this->deleteByGroup('sessions', [
$project = $dbForConsole->getDocument('projects', $project->getId()); Query::lessThan('$createdAt', $expired)
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; ], $dbForProject);
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
});
} }
/** /**
@ -601,19 +593,16 @@ class Deletes extends Action
* @return void * @return void
* @throws Exception * @throws Exception
*/ */
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAbuseRetention): void private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
{ {
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAbuseRetention) { $projectId = $project->getId();
$projectId = $project->getId(); $dbForProject = $getProjectDB($project);
$dbForProject = $getProjectDB($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$datetime = $getProjectAbuseRetention($project); $abuse = new Abuse($timeLimit);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject); $status = $abuse->cleanup($abuseRetention);
$abuse = new Abuse($timeLimit); if (!$status) {
$status = $abuse->cleanup($datetime); throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
if (!$status) { }
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
});
} }
/** /**
@ -623,18 +612,15 @@ class Deletes extends Action
* @return void * @return void
* @throws Exception * @throws Exception
*/ */
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAuditRetention): void private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void
{ {
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAuditRetention) { $projectId = $project->getId();
$projectId = $project->getId(); $dbForProject = $getProjectDB($project);
$dbForProject = $getProjectDB($project); $audit = new Audit($dbForProject);
$datetime = $getProjectAuditRetention($project); $status = $audit->cleanup($auditRetention);
$audit = new Audit($dbForProject); if (!$status) {
$status = $audit->cleanup($datetime); throw new Exception('Failed to delete Audit logs for project' . $projectId);
if (!$status) { }
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
});
} }
/** /**
@ -869,39 +855,6 @@ class Deletes extends Action
} }
} }
/**
* @param Database $dbForConsole
* @param callable $callback
* @throws Exception
*/
private function deleteForProjectIds(database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
/** /**
* @param string $collection collectionID * @param string $collection collectionID
* @param array $queries * @param array $queries

View file

@ -421,6 +421,16 @@ class Functions extends Action
$error = $th->getMessage(); $error = $th->getMessage();
$errorCode = $th->getCode(); $errorCode = $th->getCode();
} finally {
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
} }
if ($function->getAttribute('logging')) { if ($function->getAttribute('logging')) {
@ -472,15 +482,5 @@ class Functions extends Action
if (!empty($error)) { if (!empty($error)) {
throw new Exception($error, $errorCode); throw new Exception($error, $errorCode);
} }
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
} }
} }

View file

@ -6,7 +6,7 @@ use Exception;
use Utopia\App; use Utopia\App;
use Utopia\CLI\Console; use Utopia\CLI\Console;
use Utopia\DSN\DSN; use Utopia\DSN\DSN;
use Utopia\Messaging\Messages\Sms; use Utopia\Messaging\Messages\SMS;
use Utopia\Messaging\Adapters\SMS\Mock; use Utopia\Messaging\Adapters\SMS\Mock;
use Utopia\Messaging\Adapters\SMS\Msg91; use Utopia\Messaging\Adapters\SMS\Msg91;
use Utopia\Messaging\Adapters\SMS\Telesign; use Utopia\Messaging\Adapters\SMS\Telesign;

View file

@ -121,8 +121,8 @@ class Usage extends Action
} }
break; break;
case $document->getCollection() === 'databases': // databases case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS))); $collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS))); $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) { if (!empty($collections['value'])) {
$metrics[] = [ $metrics[] = [
'key' => METRIC_COLLECTIONS, 'key' => METRIC_COLLECTIONS,
@ -140,7 +140,7 @@ class Usage extends Action
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection()); $parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0; $databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS))); $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) { if (!empty($documents['value'])) {
$metrics[] = [ $metrics[] = [
@ -155,8 +155,8 @@ class Usage extends Action
break; break;
case $document->getCollection() === 'buckets': case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES))); $files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE))); $storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) { if (!empty($files['value'])) {
$metrics[] = [ $metrics[] = [
@ -174,13 +174,13 @@ class Usage extends Action
break; break;
case $document->getCollection() === 'functions': case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); $deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); $deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); $builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); $buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); $buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); $executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); $executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) { if (!empty($deployments['value'])) {
$metrics[] = [ $metrics[] = [

View file

@ -67,7 +67,7 @@ class UsageHook extends Usage
$id = \md5("{$time}_{$period}_{$key}"); $id = \md5("{$time}_{$period}_{$key}");
try { try {
$dbForProject->createDocument('stats', new Document([ $dbForProject->createDocument('stats_v2', new Document([
'$id' => $id, '$id' => $id,
'period' => $period, 'period' => $period,
'time' => $time, 'time' => $time,
@ -78,14 +78,14 @@ class UsageHook extends Usage
} catch (Duplicate $th) { } catch (Duplicate $th) {
if ($value < 0) { if ($value < 0) {
$dbForProject->decreaseDocumentAttribute( $dbForProject->decreaseDocumentAttribute(
'stats', 'stats_v2',
$id, $id,
'value', 'value',
abs($value) abs($value)
); );
} else { } else {
$dbForProject->increaseDocumentAttribute( $dbForProject->increaseDocumentAttribute(
'stats', 'stats_v2',
$id, $id,
'value', 'value',
$value $value