Merge pull request #2808 from appwrite/fix-usage-worker

Usage worker refactor and fix-document counters
This commit is contained in:
Eldad A. Fux 2022-02-20 19:17:51 +02:00 committed by GitHub
commit 041696ed5e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 430 additions and 493 deletions

View file

@ -236,6 +236,8 @@ $cli
$max = 10;
$sleep = 1;
$db = null;
$redis = null;
do { // connect to db
try {
$attempts++;
@ -275,47 +277,47 @@ $cli
* @var InfluxDB\Client $client
*/
$client = $register->get('influxdb');
if ($client) {
$attempts = 0;
$max = 10;
$sleep = 1;
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if(in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if(in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} while ($attempts < $max);
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
// sync data
foreach ($globalMetrics as $metric => $options) { //for each metrics
foreach ($periods as $period) { // aggregate data for each period
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($latestTime[$metric][$period['key']])) {
$start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339);
}
$end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339);
// sync data
foreach ($globalMetrics as $metric => $options) { //for each metrics
foreach ($periods as $period) { // aggregate data for each period
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($latestTime[$metric][$period['key']])) {
$start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339);
}
$end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339);
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
} else {
$filters = '';
}
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
} else {
$filters = '';
}
$query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)";
$query = "SELECT sum(value) AS \"value\" FROM \"{$table}\" WHERE \"time\" > '{$start}' AND \"time\" < '{$end}' AND \"metric_type\"='counter' {$filters} GROUP BY time({$period['key']}), \"projectId\" {$groupBy} FILL(null)";
try {
$result = $database->query($query);
$points = $result->getPoints();
@ -362,124 +364,305 @@ $cli
}
}
}
} catch (\Exception $e) {
Console::warning("Failed to Query: {$e->getMessage()}");
}
}
}
if ($iterations % 30 != 0) { // Aggregate aggregate number of objects in database only after 15 minutes
$iterations++;
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
return;
}
/**
* Aggregate MariaDB every 15 minutes
* Some of the queries here might contain full-table scans.
*/
if ($iterations % 30 === 0) { // Every 15 minutes aggregate number of objects in database
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating database counters.");
$latestProject = null;
do { // Loop over all the projects
$attempts = 0;
$max = 10;
$sleep = 1;
$latestProject = null;
do { // Loop over all the projects
$attempts = 0;
$max = 10;
$sleep = 1;
do { // list projects
try {
$attempts++;
$projects = $dbForConsole->find('projects', [], 100, cursor: $latestProject);
break; // leave the do-while if successful
} catch (\Exception $e) {
Console::warning("Console DB not ready yet. Retrying ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed access console db: ' . $e->getMessage());
}
sleep($sleep);
do { // list projects
try {
$attempts++;
$projects = $dbForConsole->find('projects', [], 100, cursor: $latestProject);
break; // leave the do-while if successful
} catch (\Exception $e) {
Console::warning("Console DB not ready yet. Retrying ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed access console db: ' . $e->getMessage());
}
} while ($attempts < $max);
sleep($sleep);
}
} while ($attempts < $max);
if (empty($projects)) {
continue;
if (empty($projects)) {
continue;
}
$latestProject = $projects[array_key_last($projects)];
foreach ($projects as $project) {
$projectId = $project->getId();
// Get total storage
$dbForProject->setNamespace('_project_' . $projectId);
$storageTotal = $dbForProject->sum('tags', 'size');
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_storage.tags.total'); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => '30m',
'time' => $time,
'metric' => 'storage.tags.total',
'value' => $storageTotal,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $storageTotal)
);
}
$latestProject = $projects[array_key_last($projects)];
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_storage.tags.total'); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => '1d',
'time' => $time,
'metric' => 'storage.tags.total',
'value' => $storageTotal,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $storageTotal)
);
}
foreach ($projects as $project) {
$projectId = $project->getId();
// Get total storage
$dbForProject->setNamespace('_project_' . $projectId);
$storageTotal = $dbForProject->sum('tags', 'size');
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_storage.tags.total'); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => '30m',
'time' => $time,
'metric' => 'storage.tags.total',
'value' => $storageTotal,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $storageTotal)
);
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_storage.tags.total'); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => '1d',
'time' => $time,
'metric' => 'storage.tags.total',
'value' => $storageTotal,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $storageTotal)
);
}
$collections = [
'users' => [
'namespace' => '',
],
'collections' => [
'metricPrefix' => 'database',
'namespace' => '',
'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
'documents' => [
'namespace' => '',
],
$collections = [
'users' => [
'namespace' => '',
],
'collections' => [
'metricPrefix' => 'database',
'namespace' => '',
'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
'documents' => [
'collectionPrefix' => 'collection_',
'namespace' => '',
],
],
'buckets' => [
'metricPrefix' => 'storage',
'namespace' => '',
'subCollections' => [
'files' => [
'namespace' => '',
'collectionPrefix' => 'bucket_',
'sum' => [
'field' => 'sizeOriginal'
]
],
]
],
'buckets' => [
'metricPrefix' => 'storage',
'namespace' => '',
'subCollections' => [
'files' => [
'namespace' => '',
'collectionPrefix' => 'bucket_',
'sum' => [
'field' => 'sizeOriginal'
]
],
]
];
]
];
foreach ($collections as $collection => $options) {
try {
foreach ($collections as $collection => $options) {
try {
$dbForProject->setNamespace("_project_{$projectId}");
$count = $dbForProject->count($collection);
$metricPrefix = $options['metricPrefix'] ?? '';
$metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
$subCollections = $options['subCollections'] ?? [];
if (empty($subCollections)) {
continue;
}
$latestParent = null;
$subCollectionCounts = []; //total project level count of sub collections
$subCollectionTotals = []; //total project level sum of sub collections
do { // Loop over all the parent collection document for each sub collection
$dbForProject->setNamespace("_project_{$projectId}");
$count = $dbForProject->count($collection);
$metricPrefix = $options['metricPrefix'] ?? '';
$metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
$parents = $dbForProject->find($collection, [], 100, cursor: $latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections
if (empty($parents)) {
continue;
}
$latestParent = $parents[array_key_last($parents)];
foreach ($parents as $parent) {
foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
$dbForProject->setNamespace("_project_{$projectId}");
$count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getId());
$subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
// check if sum calculation is required
$sum = $subOptions['sum'] ?? [];
if(empty($sum)) {
continue;
}
$dbForProject->setNamespace("_project_{$projectId}");
$total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getId(), $sum['field']);
$subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.total";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $total,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $total));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $total,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $total));
}
}
}
} while (!empty($parents));
/**
* Inserting project level counts for sub collections like database.documents.count
*/
foreach ($subCollectionCounts as $subCollection => $count) {
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
@ -520,227 +703,56 @@ $cli
$document->setAttribute('value', $count)
);
}
$subCollections = $options['subCollections'] ?? [];
if (empty($subCollections)) {
continue;
}
$latestParent = null;
$subCollectionCounts = []; //total project level count of sub collections
$subCollectionTotals = []; //total project level sum of sub collections
do { // Loop over all the parent collection document for each sub collection
$dbForProject->setNamespace("_project_{$projectId}");
$parents = $dbForProject->find($collection, [], 100, cursor: $latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections
if (empty($parents)) {
continue;
}
$latestParent = $parents[array_key_last($parents)];
foreach ($parents as $parent) {
foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
$dbForProject->setNamespace("_project_{$projectId}");
$count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getId());
$subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
// check if sum calculation is required
$sum = $subOptions['sum'] ?? [];
if(empty($sum)) {
continue;
}
$dbForProject->setNamespace("_project_{$projectId}");
$total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getId(), $sum['field']);
$subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.total";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $total,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $total));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $total,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $total));
}
}
}
} while (!empty($parents));
/**
* Inserting project level counts for sub collections like database.documents.count
*/
foreach ($subCollectionCounts as $subCollection => $count) {
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $count)
);
}
}
/**
* Inserting project level sums for sub collections like storage.total
*/
foreach ($subCollectionTotals as $subCollection => $count) {
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
}
} catch (\Exception$e) {
Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}");
}
/**
* Inserting project level sums for sub collections like storage.total
*/
foreach ($subCollectionTotals as $subCollection => $count) {
$dbForProject->setNamespace("_project_{$projectId}");
$metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
}
} catch (\Exception$e) {
Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}");
}
}
} while (!empty($projects));
}
}
} while (!empty($projects));
$iterations++;
$loopTook = microtime(true) - $loopStart;

View file

@ -44,11 +44,11 @@
"utopia-php/analytics": "0.2.*",
"utopia-php/audit": "0.8.*",
"utopia-php/cache": "0.4.*",
"utopia-php/cli": "0.11.*",
"utopia-php/cli": "0.12.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.14.*",
"utopia-php/locale": "0.4.*",
"utopia-php/orchestration": "0.2.*",
"utopia-php/orchestration": "0.4.*",
"utopia-php/registry": "0.5.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/domains": "1.1.*",

203
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "0a0bb53e6a5daeb3afde139b8199da3b",
"content-hash": "71d2e3bdd2ee9ed2bd4c8ae2e62ea1a2",
"packages": [
{
"name": "adhocore/jwt",
@ -531,12 +531,12 @@
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\": "src/"
},
"files": [
"src/functions_include.php"
]
],
"psr-4": {
"GuzzleHttp\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -2037,16 +2037,16 @@
},
{
"name": "utopia-php/cli",
"version": "0.11.0",
"version": "0.12.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/cli.git",
"reference": "c7a6908a8dbe9234b8b2c954e5487d34cb079af6"
"reference": "6d164b752efeb1ca089e3a517bc274d8b383474b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/c7a6908a8dbe9234b8b2c954e5487d34cb079af6",
"reference": "c7a6908a8dbe9234b8b2c954e5487d34cb079af6",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/6d164b752efeb1ca089e3a517bc274d8b383474b",
"reference": "6d164b752efeb1ca089e3a517bc274d8b383474b",
"shasum": ""
},
"require": {
@ -2084,9 +2084,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/cli/issues",
"source": "https://github.com/utopia-php/cli/tree/0.11.0"
"source": "https://github.com/utopia-php/cli/tree/0.12.0"
},
"time": "2021-04-16T15:16:08+00:00"
"time": "2022-02-18T22:10:41+00:00"
},
{
"name": "utopia-php/config",
@ -2258,16 +2258,16 @@
},
{
"name": "utopia-php/framework",
"version": "0.19.6",
"version": "0.19.7",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/framework.git",
"reference": "7d9b28365fb794001cb34dd028659452d4e71b7d"
"reference": "f17afe77a21873b9be18ebc05283813468b4283a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/framework/zipball/7d9b28365fb794001cb34dd028659452d4e71b7d",
"reference": "7d9b28365fb794001cb34dd028659452d4e71b7d",
"url": "https://api.github.com/repos/utopia-php/framework/zipball/f17afe77a21873b9be18ebc05283813468b4283a",
"reference": "f17afe77a21873b9be18ebc05283813468b4283a",
"shasum": ""
},
"require": {
@ -2301,9 +2301,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/framework/issues",
"source": "https://github.com/utopia-php/framework/tree/0.19.6"
"source": "https://github.com/utopia-php/framework/tree/0.19.7"
},
"time": "2022-02-10T17:05:22+00:00"
"time": "2022-02-18T00:04:49+00:00"
},
{
"name": "utopia-php/image",
@ -2476,21 +2476,21 @@
},
{
"name": "utopia-php/orchestration",
"version": "0.2.1",
"version": "0.4.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/orchestration.git",
"reference": "55da7a331a45d5887de8122268dfccd15fee94d1"
"reference": "67cf0ab15a096d274c093ea918aa4ace14ac7af7"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/55da7a331a45d5887de8122268dfccd15fee94d1",
"reference": "55da7a331a45d5887de8122268dfccd15fee94d1",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/67cf0ab15a096d274c093ea918aa4ace14ac7af7",
"reference": "67cf0ab15a096d274c093ea918aa4ace14ac7af7",
"shasum": ""
},
"require": {
"php": ">=8.0",
"utopia-php/cli": "0.11.*"
"utopia-php/cli": "0.12.*"
},
"require-dev": {
"phpunit/phpunit": "^9.3",
@ -2525,9 +2525,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/orchestration/issues",
"source": "https://github.com/utopia-php/orchestration/tree/0.2.1"
"source": "https://github.com/utopia-php/orchestration/tree/0.4.1"
},
"time": "2021-09-03T11:29:20+00:00"
"time": "2022-02-20T09:23:06+00:00"
},
{
"name": "utopia-php/preloader",
@ -2636,16 +2636,16 @@
},
{
"name": "utopia-php/storage",
"version": "0.6.0",
"version": "0.7.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "ad628025be3b3e0818ea1e6cb701cc08a1c52344"
"reference": "de12487f77346475e2deed82212c6f842c9ae4a8"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/ad628025be3b3e0818ea1e6cb701cc08a1c52344",
"reference": "ad628025be3b3e0818ea1e6cb701cc08a1c52344",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/de12487f77346475e2deed82212c6f842c9ae4a8",
"reference": "de12487f77346475e2deed82212c6f842c9ae4a8",
"shasum": ""
},
"require": {
@ -2682,9 +2682,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/0.6.0"
"source": "https://github.com/utopia-php/storage/tree/0.7.0"
},
"time": "2022-01-31T06:29:51+00:00"
"time": "2022-02-09T13:02:47+00:00"
},
{
"name": "utopia-php/swoole",
@ -3087,7 +3087,7 @@
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator",
"reference": "96735eef25d5a98d0582bb4925b42db593c974f8"
"reference": "b977fcf357a267f41299539ac9095aa7bbcc2600"
},
"require": {
"ext-curl": "*",
@ -3095,10 +3095,10 @@
"ext-mbstring": "*",
"matthiasmullie/minify": "^1.3",
"php": ">=7.0.0",
"twig/twig": "^2.14"
"twig/twig": "^3.3"
},
"require-dev": {
"phpunit/phpunit": "^7.0"
"phpunit/phpunit": "^9.5.13"
},
"type": "library",
"autoload": {
@ -3107,6 +3107,11 @@
"Appwrite\\Spec\\": "src/Spec"
}
},
"autoload-dev": {
"psr-4": {
"Tests\\": "tests"
}
},
"license": [
"MIT"
],
@ -3117,7 +3122,7 @@
}
],
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"time": "2022-01-30T07:29:08+00:00"
"time": "2022-02-15T11:09:40+00:00"
},
{
"name": "composer/pcre",
@ -4230,16 +4235,16 @@
},
{
"name": "phpunit/php-code-coverage",
"version": "9.2.10",
"version": "9.2.11",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/php-code-coverage.git",
"reference": "d5850aaf931743067f4bfc1ae4cbd06468400687"
"reference": "665a1ac0a763c51afc30d6d130dac0813092b17f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d5850aaf931743067f4bfc1ae4cbd06468400687",
"reference": "d5850aaf931743067f4bfc1ae4cbd06468400687",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/665a1ac0a763c51afc30d6d130dac0813092b17f",
"reference": "665a1ac0a763c51afc30d6d130dac0813092b17f",
"shasum": ""
},
"require": {
@ -4295,7 +4300,7 @@
],
"support": {
"issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.10"
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.11"
},
"funding": [
{
@ -4303,7 +4308,7 @@
"type": "github"
}
],
"time": "2021-12-05T09:12:13+00:00"
"time": "2022-02-18T12:46:09+00:00"
},
{
"name": "phpunit/php-file-iterator",
@ -5208,16 +5213,16 @@
},
{
"name": "sebastian/global-state",
"version": "5.0.4",
"version": "5.0.5",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/global-state.git",
"reference": "19c519631c5a511b7ed0ad64a6713fdb3fd25fe4"
"reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/19c519631c5a511b7ed0ad64a6713fdb3fd25fe4",
"reference": "19c519631c5a511b7ed0ad64a6713fdb3fd25fe4",
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2",
"reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2",
"shasum": ""
},
"require": {
@ -5260,7 +5265,7 @@
],
"support": {
"issues": "https://github.com/sebastianbergmann/global-state/issues",
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.4"
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5"
},
"funding": [
{
@ -5268,7 +5273,7 @@
"type": "github"
}
],
"time": "2022-02-10T07:01:19+00:00"
"time": "2022-02-14T08:28:10+00:00"
},
{
"name": "sebastian/lines-of-code",
@ -6002,12 +6007,12 @@
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Mbstring\\": ""
},
"files": [
"bootstrap.php"
]
],
"psr-4": {
"Symfony\\Polyfill\\Mbstring\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -6051,82 +6056,6 @@
],
"time": "2021-11-30T18:21:41+00:00"
},
{
"name": "symfony/polyfill-php72",
"version": "v1.24.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-php72.git",
"reference": "9a142215a36a3888e30d0a9eeea9766764e96976"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-php72/zipball/9a142215a36a3888e30d0a9eeea9766764e96976",
"reference": "9a142215a36a3888e30d0a9eeea9766764e96976",
"shasum": ""
},
"require": {
"php": ">=7.1"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.23-dev"
},
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
}
},
"autoload": {
"files": [
"bootstrap.php"
],
"psr-4": {
"Symfony\\Polyfill\\Php72\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony polyfill backporting some PHP 7.2+ features to lower PHP versions",
"homepage": "https://symfony.com",
"keywords": [
"compatibility",
"polyfill",
"portable",
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-php72/tree/v1.24.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2021-05-27T09:17:38+00:00"
},
{
"name": "symfony/service-contracts",
"version": "v3.0.0",
@ -6395,23 +6324,22 @@
},
{
"name": "twig/twig",
"version": "v2.14.11",
"version": "v3.3.8",
"source": {
"type": "git",
"url": "https://github.com/twigphp/Twig.git",
"reference": "66baa66f29ee30e487e05f1679903e36eb01d727"
"reference": "972d8604a92b7054828b539f2febb0211dd5945c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/twigphp/Twig/zipball/66baa66f29ee30e487e05f1679903e36eb01d727",
"reference": "66baa66f29ee30e487e05f1679903e36eb01d727",
"url": "https://api.github.com/repos/twigphp/Twig/zipball/972d8604a92b7054828b539f2febb0211dd5945c",
"reference": "972d8604a92b7054828b539f2febb0211dd5945c",
"shasum": ""
},
"require": {
"php": ">=7.1.3",
"php": ">=7.2.5",
"symfony/polyfill-ctype": "^1.8",
"symfony/polyfill-mbstring": "^1.3",
"symfony/polyfill-php72": "^1.8"
"symfony/polyfill-mbstring": "^1.3"
},
"require-dev": {
"psr/container": "^1.0",
@ -6420,13 +6348,10 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.14-dev"
"dev-master": "3.3-dev"
}
},
"autoload": {
"psr-0": {
"Twig_": "lib/"
},
"psr-4": {
"Twig\\": "src/"
}
@ -6459,7 +6384,7 @@
],
"support": {
"issues": "https://github.com/twigphp/Twig/issues",
"source": "https://github.com/twigphp/Twig/tree/v2.14.11"
"source": "https://github.com/twigphp/Twig/tree/v3.3.8"
},
"funding": [
{
@ -6471,7 +6396,7 @@
"type": "tidelift"
}
],
"time": "2022-02-04T06:57:25+00:00"
"time": "2022-02-04T06:59:48+00:00"
},
{
"name": "vimeo/psalm",