mirror of
https://github.com/appwrite/appwrite
synced 2026-05-23 00:49:02 +00:00
Merge remote-tracking branch 'origin/1.7.x' into 1.8.x
# Conflicts: # src/Appwrite/Platform/Workers/StatsUsageDump.php
This commit is contained in:
Jake Barnby
commit
7dd45b0644
6 changed files with 38 additions and 218 deletions
|
|
@ -59,7 +59,7 @@ return [
|
|||
[
|
||||
'key' => 'flutter',
|
||||
'name' => 'Flutter',
|
||||
'version' => '16.1.0',
|
||||
'version' => '17.0.1',
|
||||
'url' => 'https://github.com/appwrite/sdk-for-flutter',
|
||||
'package' => 'https://pub.dev/packages/appwrite',
|
||||
'enabled' => true,
|
||||
|
|
@ -77,7 +77,7 @@ return [
|
|||
[
|
||||
'key' => 'apple',
|
||||
'name' => 'Apple',
|
||||
'version' => '10.1.0',
|
||||
'version' => '10.1.1',
|
||||
'url' => 'https://github.com/appwrite/sdk-for-apple',
|
||||
'package' => 'https://github.com/appwrite/sdk-for-apple',
|
||||
'enabled' => true,
|
||||
|
|
@ -217,7 +217,7 @@ return [
|
|||
[
|
||||
'key' => 'cli',
|
||||
'name' => 'Command Line',
|
||||
'version' => '7.0.0',
|
||||
'version' => '8.0.0',
|
||||
'url' => 'https://github.com/appwrite/sdk-for-cli',
|
||||
'package' => 'https://www.npmjs.com/package/appwrite-cli',
|
||||
'enabled' => true,
|
||||
|
|
@ -231,6 +231,11 @@ return [
|
|||
'gitRepoName' => 'sdk-for-cli',
|
||||
'gitUserName' => 'appwrite',
|
||||
'gitBranch' => 'dev',
|
||||
'exclude' => [
|
||||
'services' => [
|
||||
['name' => 'assistant'],
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
|
|
@ -411,7 +416,7 @@ return [
|
|||
[
|
||||
'key' => 'swift',
|
||||
'name' => 'Swift',
|
||||
'version' => '10.0.0',
|
||||
'version' => '10.1.0',
|
||||
'url' => 'https://github.com/appwrite/sdk-for-swift',
|
||||
'package' => 'https://github.com/appwrite/sdk-for-swift',
|
||||
'enabled' => true,
|
||||
|
|
|
|||
|
|
@ -845,15 +845,18 @@ App::get('/v1/health/storage')
|
|||
$checkStart = \microtime(true);
|
||||
|
||||
foreach ($devices as $device) {
|
||||
if (!$device->write($device->getPath('health.txt'), 'test', 'text/plain')) {
|
||||
$uniqueFileName = \uniqid('health', true);
|
||||
$filePath = $device->getPath($uniqueFileName);
|
||||
|
||||
if (!$device->write($filePath, 'test', 'text/plain')) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed writing test file to ' . $device->getRoot());
|
||||
}
|
||||
|
||||
if ($device->read($device->getPath('health.txt')) !== 'test') {
|
||||
if ($device->read($filePath) !== 'test') {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed reading test file from ' . $device->getRoot());
|
||||
}
|
||||
|
||||
if (!$device->delete($device->getPath('health.txt'))) {
|
||||
if (!$device->delete($filePath)) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed deleting test file from ' . $device->getRoot());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -877,6 +877,10 @@ class Builds extends Action
|
|||
}
|
||||
}
|
||||
|
||||
$deployment->setAttribute('buildLogs', $logs);
|
||||
|
||||
$this->afterBuildSuccess($dbForProject, $deployment);
|
||||
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
|
||||
|
||||
$queueForRealtime
|
||||
|
|
@ -1310,6 +1314,19 @@ class Builds extends Action
|
|||
->trigger();
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to run after build success
|
||||
*
|
||||
* @param Database $dbForProject
|
||||
* @param Document $deployment
|
||||
* @return void
|
||||
*/
|
||||
protected function afterBuildSuccess(Database $dbForProject, Document &$deployment): void
|
||||
{
|
||||
assert($dbForProject instanceof Database);
|
||||
assert($deployment instanceof Document);
|
||||
}
|
||||
|
||||
protected function getRuntime(Document $resource, string $version): array
|
||||
{
|
||||
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
|
||||
|
|
|
|||
|
|
@ -64,9 +64,9 @@ class Maintenance extends Action
|
|||
|
||||
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
|
||||
|
||||
// Iterate through project only if it was accessed in last 24 hours
|
||||
$dateInterval = DateInterval::createFromDateString('24 hours');
|
||||
$before24h = (new DateTime())->sub($dateInterval);
|
||||
// Iterate through project only if it was accessed in last 30 days
|
||||
$dateInterval = DateInterval::createFromDateString('30 days');
|
||||
$before30days = (new DateTime())->sub($dateInterval);
|
||||
|
||||
$dbForPlatform->foreach(
|
||||
'projects',
|
||||
|
|
@ -80,7 +80,7 @@ class Maintenance extends Action
|
|||
[
|
||||
Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
|
||||
Query::limit(100),
|
||||
Query::greaterThanEqual('accessedAt', DatabaseDateTime::format($before24h)),
|
||||
Query::greaterThanEqual('accessedAt', DatabaseDateTime::format($before30days)),
|
||||
Query::orderAsc('teamInternalId'),
|
||||
]
|
||||
);
|
||||
|
|
|
|||
|
|
@ -251,7 +251,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
|||
->setDiscord(APP_SOCIAL_DISCORD_CHANNEL, APP_SOCIAL_DISCORD)
|
||||
->setDefaultHeaders([
|
||||
'X-Appwrite-Response-Format' => '1.7.0',
|
||||
]);
|
||||
])
|
||||
->setExclude($language['exclude'] ?? []);
|
||||
|
||||
// Make sure we have a clean slate.
|
||||
// Otherwise, all files in this dir will be pushed,
|
||||
|
|
|
|||
|
|
@ -1,206 +0,0 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Extend\Exception;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\Registry\Registry;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* TODO remove later
|
||||
*/
|
||||
class StatsUsageDump extends Action
|
||||
{
|
||||
public const METRIC_COLLECTION_LEVEL_STORAGE = 4;
|
||||
public const METRIC_DATABASE_LEVEL_STORAGE = 3;
|
||||
public const METRIC_PROJECT_LEVEL_STORAGE = 2;
|
||||
protected array $stats = [];
|
||||
|
||||
protected Registry $register;
|
||||
|
||||
/**
|
||||
* Metrics to skip writing to logsDB
|
||||
* As these metrics are calculated separately
|
||||
* by logs DB
|
||||
* @var array
|
||||
*/
|
||||
protected array $skipBaseMetrics = [
|
||||
METRIC_DATABASES => true,
|
||||
METRIC_BUCKETS => true,
|
||||
METRIC_USERS => true,
|
||||
METRIC_FUNCTIONS => true,
|
||||
METRIC_TEAMS => true,
|
||||
METRIC_MESSAGES => true,
|
||||
METRIC_MAU => true,
|
||||
METRIC_WEBHOOKS => true,
|
||||
METRIC_PLATFORMS => true,
|
||||
METRIC_PROVIDERS => true,
|
||||
METRIC_TOPICS => true,
|
||||
METRIC_KEYS => true,
|
||||
METRIC_FILES => true,
|
||||
METRIC_FILES_STORAGE => true,
|
||||
METRIC_DEPLOYMENTS_STORAGE => true,
|
||||
METRIC_BUILDS_STORAGE => true,
|
||||
METRIC_DEPLOYMENTS => true,
|
||||
METRIC_BUILDS => true,
|
||||
METRIC_COLLECTIONS => true,
|
||||
METRIC_DOCUMENTS => true,
|
||||
METRIC_DATABASES_STORAGE => true,
|
||||
];
|
||||
|
||||
/**
|
||||
* Skip metrics associated with parent IDs
|
||||
* these need to be checked individually with `str_ends_with`
|
||||
*/
|
||||
protected array $skipParentIdMetrics = [
|
||||
'.files',
|
||||
'.files.storage',
|
||||
'.collections',
|
||||
'.documents',
|
||||
'.deployments',
|
||||
'.deployments.storage',
|
||||
'.builds',
|
||||
'.builds.storage',
|
||||
'.databases.storage'
|
||||
];
|
||||
|
||||
/**
|
||||
* @var callable(Document): Database
|
||||
*/
|
||||
protected $getLogsDB;
|
||||
|
||||
protected array $periods = [
|
||||
'1h' => 'Y-m-d H:00',
|
||||
'1d' => 'Y-m-d 00:00',
|
||||
'inf' => '0000-00-00 00:00'
|
||||
];
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'stats-usage-dump';
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->inject('message')
|
||||
->inject('getProjectDB')
|
||||
->inject('getLogsDB')
|
||||
->inject('register')
|
||||
->callback([$this, 'action']);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Message $message
|
||||
* @param callable $getProjectDB
|
||||
* @param callable $getLogsDB
|
||||
* @param Registry $register
|
||||
* @return void
|
||||
* @throws Exception
|
||||
* @throws \Throwable
|
||||
* @throws \Utopia\Database\Exception
|
||||
*/
|
||||
public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void
|
||||
{
|
||||
$this->getLogsDB = $getLogsDB;
|
||||
$this->register = $register;
|
||||
$payload = $message->getPayload() ?? [];
|
||||
if (empty($payload)) {
|
||||
throw new Exception('Missing payload');
|
||||
}
|
||||
|
||||
foreach ($payload['stats'] ?? [] as $stats) {
|
||||
$project = new Document($stats['project'] ?? []);
|
||||
|
||||
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
|
||||
$receivedAt = $stats['receivedAt'] ?? null;
|
||||
if ($numberOfKeys === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getSequence(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys);
|
||||
|
||||
try {
|
||||
/** @var Database $dbForProject */
|
||||
$dbForProject = $getProjectDB($project);
|
||||
foreach ($stats['keys'] ?? [] as $key => $value) {
|
||||
if ($value == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period => $format) {
|
||||
$time = null;
|
||||
|
||||
if ($period !== 'inf') {
|
||||
$time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time());
|
||||
}
|
||||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
|
||||
$document = new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
'metric' => $key,
|
||||
'value' => $value,
|
||||
'region' => System::getEnv('_APP_REGION', 'default'),
|
||||
]);
|
||||
|
||||
$documentClone = clone $document;
|
||||
|
||||
$dbForProject->createOrUpdateDocumentsWithIncrease(
|
||||
'stats',
|
||||
'value',
|
||||
[$document]
|
||||
);
|
||||
|
||||
$this->writeToLogsDB($project, $documentClone);
|
||||
}
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
Console::error('[' . DateTime::now() . '] project [' . $project->getSequence() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected function writeToLogsDB(Document $project, Document $document): void
|
||||
{
|
||||
if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') {
|
||||
Console::log('Dual Writing is disabled. Skipping...');
|
||||
return;
|
||||
}
|
||||
|
||||
if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) {
|
||||
return;
|
||||
}
|
||||
foreach ($this->skipParentIdMetrics as $skipMetric) {
|
||||
if (str_ends_with($document->getAttribute('metric'), $skipMetric)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
$dbForLogs = ($this->getLogsDB)($project);
|
||||
|
||||
try {
|
||||
$dbForLogs->createOrUpdateDocumentsWithIncrease(
|
||||
'stats',
|
||||
'value',
|
||||
[$document]
|
||||
);
|
||||
Console::success('Usage logs pushed to Logs DB');
|
||||
} catch (\Throwable $th) {
|
||||
Console::error($th->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue