From 9f537a5beebed1434a6f9e1c532f714882932e84 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 27 Mar 2025 04:59:34 +0000 Subject: [PATCH 01/29] Feat: multi tenant dual writing --- composer.json | 2 +- composer.lock | 66 ++--- .../Platform/Workers/StatsUsageDump.php | 263 ++++++------------ tests/e2e/General/UsageTest.php | 110 ++++---- 4 files changed, 177 insertions(+), 264 deletions(-) diff --git a/composer.json b/composer.json index b1b6aed539..b8c5afa109 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.62.*", + "utopia-php/database": "0.63.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 1ea887fdbc..bd9366a34c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d2bec8137dcd84994121f89a29932d31", + "content-hash": "5b3c46863e4571c838c30090ad96436c", "packages": [ { "name": "adhocore/jwt", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.1", + "version": "v4.30.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/a4c4d8565b40b9f76debc9dfeb221412eacb8ced", + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.2" }, - "time": "2025-03-13T21:08:17+00:00" + "time": "2025-03-26T18:01:50+00:00" }, { "name": "league/csv", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.62.1", + "version": "0.63.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "65dc51466c12552add10395900cdbb4728da4068" + "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/65dc51466c12552add10395900cdbb4728da4068", - "reference": "65dc51466c12552add10395900cdbb4728da4068", + "url": "https://api.github.com/repos/utopia-php/database/zipball/51ff0a6514e6eda44a6da016d838e7e55282e01a", + "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.62.1" + "source": "https://github.com/utopia-php/database/tree/0.63.0" }, - "time": "2025-03-24T08:27:18+00:00" + "time": "2025-03-26T10:35:37+00:00" }, { "name": "utopia-php/domains", @@ -3950,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.2", + "version": "0.8.3", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1" + "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", - "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", "shasum": "" }, "require": { @@ -3967,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.62.*", + "utopia-php/database": "0.63.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.2" + "source": "https://github.com/utopia-php/migration/tree/0.8.3" }, - "time": "2025-03-24T09:05:31+00:00" + "time": "2025-03-26T10:45:51+00:00" }, { "name": "utopia-php/orchestration", @@ -4385,16 +4385,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.8.2", + "version": "0.8.3", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4" + "reference": "1af73dd3e73987cf729c7db399054e4a70befd99" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4", - "reference": "5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/1af73dd3e73987cf729c7db399054e4a70befd99", + "reference": "1af73dd3e73987cf729c7db399054e4a70befd99", "shasum": "" }, "require": { @@ -4430,9 +4430,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.8.2" + "source": "https://github.com/utopia-php/swoole/tree/0.8.3" }, - "time": "2024-02-01T14:54:12+00:00" + "time": "2025-03-26T10:09:05+00:00" }, { "name": "utopia-php/system", @@ -4775,16 +4775,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.9", + "version": "0.40.10", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4" + "reference": "054ac96285caf4f77879087b2416a5ddb8263051" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/dbb45a5db22cdc3368fe2573c07ba6088f188fa4", - "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", + "reference": "054ac96285caf4f77879087b2416a5ddb8263051", "shasum": "" }, "require": { @@ -4820,9 +4820,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.9" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" }, - "time": "2025-03-17T18:39:14+00:00" + "time": "2025-03-25T13:44:16+00:00" }, { "name": "doctrine/annotations", @@ -8134,7 +8134,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 119a9e7288..9b7c119821 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Extend\Exception; +use Throwable; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -14,11 +15,29 @@ use Utopia\System\System; class StatsUsageDump extends Action { - public const METRIC_COLLECTION_LEVEL_STORAGE = 4; - public const METRIC_DATABASE_LEVEL_STORAGE = 3; - public const METRIC_PROJECT_LEVEL_STORAGE = 2; + protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + + private int $lastDispatchTime = 0; + private int $lastDispatchTimeLogsDB = 0; protected array $stats = []; + /** + * Stats for batch write separated per project + * @var array + */ + private array $projects = []; + + /** + * Array of stat documents to batch write to logsDB + * @var array + */ + private array $statDocuments = []; + + protected function getBatchSize(): int + { + return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); + } + protected Registry $register; /** @@ -127,19 +146,13 @@ class StatsUsageDump extends Action console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); try { - /** @var \Utopia\Database\Database $dbForProject */ - $dbForProject = $getProjectDB($project); foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; } if (str_contains($key, METRIC_DATABASES_STORAGE)) { - try { - $this->handleDatabaseStorage($key, $dbForProject, $project, $receivedAt); - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); - } + // skip database storage calc as it's wrong and we plan to get this from StatsResources continue; } @@ -160,202 +173,102 @@ class StatsUsageDump extends Action 'region' => System::getEnv('_APP_REGION', 'default'), ]); - $documentClone = new Document($document->getArrayCopy()); + $this->projects[$project->getInternalId()]['project'] = new Document([ + '$id' => $project->getId(), + '$internalId' => $project->getInternalId(), + 'database' => $project->getAttribute('database'), + ]); + $this->projects[$project->getInternalId()]['stats'] = $document; - $dbForProject->createOrUpdateDocumentsWithIncrease( - 'stats', - 'value', - [$document] - ); - - $this->writeToLogsDB($project, $documentClone); + $this->prepareForLogsDB($project, $document); } } } catch (\Exception $e) { console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); } } - } - private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project, string $receivedAt): void - { - $data = explode('.', $key); - $start = microtime(true); - - $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) use ($receivedAt) { - $id = \md5("{$time}_{$period}_{$key}"); - - $document = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - $documentClone = new Document($document->getArrayCopy()); - $dbForProject->createOrUpdateDocumentsWithIncrease( - 'stats', - 'value', - [$document] - ); - $this->writeToLogsDB($project, $documentClone); - }; - - foreach ($this->periods as $period => $format) { - $time = null; - - if ($period !== 'inf') { - $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); - } - $id = \md5("{$time}_{$period}_{$key}"); - - $value = 0; - $previousValue = 0; - try { - $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); - } catch (\Exception $e) { - // No previous value - } - - switch (count($data)) { - // Collection Level - case self::METRIC_COLLECTION_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; - - try { - $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - // Compare with previous value - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Collection - $updateMetric($dbForProject, $project, $diff, $key, $period, $time); - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - // Database Level - case self::METRIC_DATABASE_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - - $collections = []; - try { - $collections = $dbForProject->find('database_' . $databaseInternalId); - } catch (\Exception $e) { - // Database not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - // Project Level - case self::METRIC_PROJECT_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); - // Get all project databases - $databases = $dbForProject->find('database'); - - // Recalculate all databases - foreach ($databases as $database) { - $collections = $dbForProject->find('database_' . $database->getInternalId()); - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - } - - $diff = $value - $previousValue; - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time); - break; - } + $batchSize = $this->getBatchSize(); + $shouldProcessBatch = \count($this->projects) >= $batchSize; + if (!$shouldProcessBatch && \count($this->projects) > 0) { + $shouldProcessBatch = (\time() - $this->lastDispatchTime) >= self::BATCH_AGGREGATION_INTERVAL; } - $end = microtime(true); + if ($shouldProcessBatch) { + try { + foreach ($this->projects as $internalId => $projectStats) { + /** @var \Utopia\Database\Database $dbForProject */ + $dbForProject = $getProjectDB($projectStats['project']); + Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); + $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); + Console::success('Batch successfully written to DB'); + + unset($this->projects[$internalId]); + } + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + $this->lastDispatchTime = time(); + } + } + $this->writeToLogsDB(); - console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); } - protected function writeToLogsDB(Document $project, Document $document): void + protected function prepareForLogsDB(Document $project, Document $stat) + { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { + Console::log('Dual Writing is disabled. Skipping...'); + return; + } + if (array_key_exists($stat->getAttribute('metric'), $this->skipBaseMetrics)) { + return; + } + foreach ($this->skipParentIdMetrics as $skipMetric) { + if (str_ends_with($stat->getAttribute('metric'), $skipMetric)) { + return; + } + } + $documentClone = new Document($stat->getArrayCopy()); + $documentClone->setAttribute('$tenant', (int) $project->getInternalId()); + $this->statDocuments[] = $documentClone; + } + + protected function writeToLogsDB(): void { if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { Console::log('Dual Writing is disabled. Skipping...'); return; } - if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) { - return; + $batchSize = $this->getBatchSize(); + $shouldProcessBatch = \count($this->statDocuments) >= $batchSize; + if (!$shouldProcessBatch && \count($this->statDocuments) > 0) { + $shouldProcessBatch = (\time() - $this->lastDispatchTimeLogsDB) >= self::BATCH_AGGREGATION_INTERVAL; } - foreach ($this->skipParentIdMetrics as $skipMetric) { - if (str_ends_with($document->getAttribute('metric'), $skipMetric)) { - return; - } + + if (!$shouldProcessBatch) { + return; } /** @var \Utopia\Database\Database $dbForLogs*/ - $dbForLogs = call_user_func($this->getLogsDB, $project); + $dbForLogs = call_user_func($this->getLogsDB); + $dbForLogs + ->setTenant(null) + ->setTenantPerDocument(true); try { + Console::log('Processing batch with ' . count($this->statDocuments) . ' stats'); $dbForLogs->createOrUpdateDocumentsWithIncrease( 'stats', 'value', - [$document] + $this->statDocuments ); Console::success('Usage logs pushed to Logs DB'); - } catch (\Throwable $th) { + } catch (Throwable $th) { Console::error($th->getMessage()); + } finally { + $this->lastDispatchTimeLogsDB = time(); } $this->register->get('pools')->get('logs')->reclaim(); diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index e614e2e185..2cceeeae05 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -593,66 +593,66 @@ class UsageTest extends Scope return $data; } - public function testDatabaseStoragePrepare(): array - { - $response = $this->client->call( - Client::METHOD_POST, - '/databases', - array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), - [ - 'databaseId' => 'unique()', - 'name' => 'dbStorageStats', - ] - ); + // public function testDatabaseStoragePrepare(): array + // { + // $response = $this->client->call( + // Client::METHOD_POST, + // '/databases', + // array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'databaseId' => 'unique()', + // 'name' => 'dbStorageStats', + // ] + // ); - $this->assertNotEmpty($response['body']['$id']); - $databaseId = $response['body']['$id']; + // $this->assertNotEmpty($response['body']['$id']); + // $databaseId = $response['body']['$id']; - $response = $this->client->call( - Client::METHOD_POST, - '/databases/' . $databaseId . '/collections', - array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), - [ - 'collectionId' => 'unique()', - 'name' => 'collectionStorageStats', - 'documentSecurity' => false, - 'permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ] - ); + // $response = $this->client->call( + // Client::METHOD_POST, + // '/databases/' . $databaseId . '/collections', + // array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'collectionId' => 'unique()', + // 'name' => 'collectionStorageStats', + // 'documentSecurity' => false, + // 'permissions' => [ + // Permission::read(Role::any()), + // Permission::create(Role::any()), + // Permission::update(Role::any()), + // Permission::delete(Role::any()), + // ], + // ] + // ); - $this->assertNotEmpty($response['body']['$id']); - $collectionId = $response['body']['$id']; + // $this->assertNotEmpty($response['body']['$id']); + // $collectionId = $response['body']['$id']; - $response = $this->client->call( - Client::METHOD_POST, - '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', - array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), - [ - 'key' => 'data', - 'size' => 100000, - 'required' => true, - ] - ); + // $response = $this->client->call( + // Client::METHOD_POST, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', + // array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'key' => 'data', + // 'size' => 100000, + // 'required' => true, + // ] + // ); - return [ - 'databaseId' => $databaseId, - 'collectionId' => $collectionId, - ]; - } + // return [ + // 'databaseId' => $databaseId, + // 'collectionId' => $collectionId, + // ]; + // } // /** @depends testDatabaseStoragePrepare */ // #[Retry(count: 1)] From c443a2f9995a60ecd0c2e50475b16be57996cf57 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 27 Mar 2025 05:50:01 +0000 Subject: [PATCH 02/29] Fix dual writing --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 9b7c119821..e6071aa185 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Extend\Exception; use Throwable; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -178,7 +179,7 @@ class StatsUsageDump extends Action '$internalId' => $project->getInternalId(), 'database' => $project->getAttribute('database'), ]); - $this->projects[$project->getInternalId()]['stats'] = $document; + $this->projects[$project->getInternalId()]['stats'][] = $document; $this->prepareForLogsDB($project, $document); } @@ -194,7 +195,7 @@ class StatsUsageDump extends Action $shouldProcessBatch = (\time() - $this->lastDispatchTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if ($shouldProcessBatch) { + if ($shouldProcessBatch || App::isDevelopment()) { try { foreach ($this->projects as $internalId => $projectStats) { /** @var \Utopia\Database\Database $dbForProject */ @@ -206,7 +207,7 @@ class StatsUsageDump extends Action unset($this->projects[$internalId]); } } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); + Console::error('Error processing stats: ' . $e->getMessage()); } finally { $this->lastDispatchTime = time(); } From 1a54bd58a6bf8317b58799b0e18a8724ae88acb5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 27 Mar 2025 07:01:44 +0000 Subject: [PATCH 03/29] check empty project id --- .../Platform/Workers/StatsUsageDump.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index e6071aa185..2832e97000 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -137,7 +137,6 @@ class StatsUsageDump extends Action foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); - $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; $receivedAt = $stats['receivedAt'] ?? null; if ($numberOfKeys === 0) { @@ -174,6 +173,7 @@ class StatsUsageDump extends Action 'region' => System::getEnv('_APP_REGION', 'default'), ]); + $this->projects[$project->getInternalId()]['project'] = new Document([ '$id' => $project->getId(), '$internalId' => $project->getInternalId(), @@ -196,8 +196,11 @@ class StatsUsageDump extends Action } if ($shouldProcessBatch || App::isDevelopment()) { - try { - foreach ($this->projects as $internalId => $projectStats) { + foreach ($this->projects as $internalId => $projectStats) { + if (empty($internalId)) { + continue; + } + try { /** @var \Utopia\Database\Database $dbForProject */ $dbForProject = $getProjectDB($projectStats['project']); Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); @@ -205,12 +208,11 @@ class StatsUsageDump extends Action Console::success('Batch successfully written to DB'); unset($this->projects[$internalId]); + } catch (Throwable $e) { + Console::error('Error processing stats: ' . $e->getMessage()); } - } catch (Throwable $e) { - Console::error('Error processing stats: ' . $e->getMessage()); - } finally { - $this->lastDispatchTime = time(); } + $this->lastDispatchTime = time(); } $this->writeToLogsDB(); @@ -219,7 +221,6 @@ class StatsUsageDump extends Action protected function prepareForLogsDB(Document $project, Document $stat) { if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { - Console::log('Dual Writing is disabled. Skipping...'); return; } if (array_key_exists($stat->getAttribute('metric'), $this->skipBaseMetrics)) { From 05067302d2d61f49d92095e48e8ed31178e1805a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 27 Mar 2025 08:25:48 +0000 Subject: [PATCH 04/29] Fix: merge the working of StatsUsage and StatsUsageDump --- src/Appwrite/Platform/Workers/StatsUsage.php | 214 ++++++++++++++++++- 1 file changed, 207 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index a755f723a0..6439fa5fbc 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -2,17 +2,21 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\StatsUsageDump; use Exception; +use Throwable; use Utopia\CLI\Console; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Registry\Registry; use Utopia\System\System; class StatsUsage extends Action { + /** + * In memory per project metrics calculation + */ private array $stats = []; private int $lastTriggeredTime = 0; private int $keys = 0; @@ -20,6 +24,77 @@ class StatsUsage extends Action private const BATCH_SIZE_DEVELOPMENT = 1; private const BATCH_SIZE_PRODUCTION = 10_000; + /** + * Stats for batch write separated per project + * @var array + */ + private array $projects = []; + + /** + * Array of stat documents to batch write to logsDB + * @var array + */ + private array $statDocuments = []; + + protected Registry $register; + + /** + * Metrics to skip writing to logsDB + * As these metrics are calculated separately + * by logs DB + * @var array + */ + protected array $skipBaseMetrics = [ + METRIC_DATABASES => true, + METRIC_BUCKETS => true, + METRIC_USERS => true, + METRIC_FUNCTIONS => true, + METRIC_TEAMS => true, + METRIC_MESSAGES => true, + METRIC_MAU => true, + METRIC_WEBHOOKS => true, + METRIC_PLATFORMS => true, + METRIC_PROVIDERS => true, + METRIC_TOPICS => true, + METRIC_KEYS => true, + METRIC_FILES => true, + METRIC_FILES_STORAGE => true, + METRIC_DEPLOYMENTS_STORAGE => true, + METRIC_BUILDS_STORAGE => true, + METRIC_DEPLOYMENTS => true, + METRIC_BUILDS => true, + METRIC_COLLECTIONS => true, + METRIC_DOCUMENTS => true, + METRIC_DATABASES_STORAGE => true, + ]; + + /** + * Skip metrics associated with parent IDs + * these need to be checked individually with `str_ends_with` + */ + protected array $skipParentIdMetrics = [ + '.files', + '.files.storage', + '.collections', + '.documents', + '.deployments', + '.deployments.storage', + '.builds', + '.builds.storage', + '.databases.storage' + ]; + + /** + * @var callable + */ + protected mixed $getLogsDB; + + protected array $periods = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + public static function getName(): string { return 'stats-usage'; @@ -41,7 +116,8 @@ class StatsUsage extends Action ->desc('Stats usage worker') ->inject('message') ->inject('getProjectDB') - ->inject('queueForStatsUsageDump') + ->inject('getLogsDB') + ->inject('register') ->callback([$this, 'action']); $this->lastTriggeredTime = time(); @@ -50,13 +126,16 @@ class StatsUsage extends Action /** * @param Message $message * @param callable $getProjectDB - * @param StatsUsageDump $queueForStatsUsageDump + * @param callable $getLogsDB + * @param Registry $register * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump): void + public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void { + $this->getLogsDB = $getLogsDB; + $this->register = $register; $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); @@ -98,9 +177,7 @@ class StatsUsage extends Action ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); - $queueForStatsUsageDump - ->setStats($this->stats) - ->trigger(); + $this->commitToDB($getProjectDB); $this->stats = []; $this->keys = 0; @@ -250,4 +327,127 @@ class StatsUsage extends Action console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } + + public function commitToDb(callable $getProjectDB): void + { + + foreach ($this->stats as $stats) { + $project = new Document($stats['project'] ?? []); + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + $receivedAt = $stats['receivedAt'] ?? null; + if ($numberOfKeys === 0) { + continue; + } + + console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + + try { + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + if (str_contains($key, METRIC_DATABASES_STORAGE)) { + // skip database storage calc as it's wrong and we plan to get this from StatsResources + continue; + } + + foreach ($this->periods as $period => $format) { + $time = null; + + if ($period !== 'inf') { + $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + } + $id = \md5("{$time}_{$period}_{$key}"); + + $document = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + + + $this->projects[$project->getInternalId()]['project'] = new Document([ + '$id' => $project->getId(), + '$internalId' => $project->getInternalId(), + 'database' => $project->getAttribute('database'), + ]); + $this->projects[$project->getInternalId()]['stats'][] = $document; + + $this->prepareForLogsDB($project, $document); + } + } + } catch (Exception $e) { + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); + } + } + + foreach ($this->projects as $internalId => $projectStats) { + if (empty($internalId)) { + continue; + } + try { + /** @var \Utopia\Database\Database $dbForProject */ + $dbForProject = $getProjectDB($projectStats['project']); + Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); + $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); + Console::success('Batch successfully written to DB'); + + unset($this->projects[$internalId]); + } catch (Throwable $e) { + Console::error('Error processing stats: ' . $e->getMessage()); + } + } + + $this->writeToLogsDB(); + + } + + protected function prepareForLogsDB(Document $project, Document $stat) + { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { + return; + } + if (array_key_exists($stat->getAttribute('metric'), $this->skipBaseMetrics)) { + return; + } + foreach ($this->skipParentIdMetrics as $skipMetric) { + if (str_ends_with($stat->getAttribute('metric'), $skipMetric)) { + return; + } + } + $documentClone = new Document($stat->getArrayCopy()); + $documentClone->setAttribute('$tenant', (int) $project->getInternalId()); + $this->statDocuments[] = $documentClone; + } + + protected function writeToLogsDB(): void + { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { + Console::log('Dual Writing is disabled. Skipping...'); + return; + } + + /** @var \Utopia\Database\Database $dbForLogs*/ + $dbForLogs = call_user_func($this->getLogsDB); + $dbForLogs + ->setTenant(null) + ->setTenantPerDocument(true); + + try { + Console::log('Processing batch with ' . count($this->statDocuments) . ' stats'); + $dbForLogs->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + $this->statDocuments + ); + Console::success('Usage logs pushed to Logs DB'); + } catch (Throwable $th) { + Console::error($th->getMessage()); + } + $this->register->get('pools')->get('logs')->reclaim(); + } } From cfb1f4fcdb6930fda09d15aa7429de34829022af Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 27 Mar 2025 08:29:12 +0000 Subject: [PATCH 05/29] fix: remove stats usage dump --- app/controllers/api/health.php | 32 -- app/worker.php | 5 - src/Appwrite/Event/Event.php | 11 - src/Appwrite/Event/StatsUsageDump.php | 44 --- src/Appwrite/Platform/Services/Workers.php | 2 - .../Platform/Workers/StatsUsageDump.php | 278 ------------------ .../Health/HealthCustomServerTest.php | 24 -- 7 files changed, 396 deletions(-) delete mode 100644 src/Appwrite/Event/StatsUsageDump.php delete mode 100644 src/Appwrite/Platform/Workers/StatsUsageDump.php diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index e5336067c8..204ba0ad0a 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -756,38 +756,6 @@ App::get('/v1/health/queue/stats-usage') $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }); -App::get('/v1/health/queue/stats-usage-dump') - ->desc('Get usage dump queue') - ->groups(['api', 'health']) - ->label('scope', 'health.read') - ->label('sdk', new Method( - auth: [AuthType::KEY], - namespace: 'health', - name: 'getQueueStatsUsageDump', - description: '/docs/references/health/get-queue-stats-usage-dump.md', - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_HEALTH_QUEUE, - ) - ], - contentType: ContentType::JSON - )) - ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('publisher') - ->inject('response') - ->action(function (int|string $threshold, Publisher $publisher, Response $response) { - $threshold = \intval($threshold); - - $size = $publisher->getQueueSize(new Queue(Event::STATS_USAGE_DUMP_QUEUE_NAME)); - - if ($size >= $threshold) { - throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); - } - - $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); - }); - App::get('/v1/health/storage/local') ->desc('Get local storage') ->groups(['api', 'health']) diff --git a/app/worker.php b/app/worker.php index 90496c0430..e061a6a6d6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -15,7 +15,6 @@ use Appwrite\Event\Messaging; use Appwrite\Event\Migration; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; -use Appwrite\Event\StatsUsageDump; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Swoole\Runtime; @@ -278,10 +277,6 @@ Server::setResource('queueForStatsUsage', function (Publisher $publisher) { return new StatsUsage($publisher); }, ['publisher']); -Server::setResource('queueForStatsUsageDump', function (Publisher $publisher) { - return new StatsUsageDump($publisher); -}, ['publisher']); - Server::setResource('queueForDatabase', function (Publisher $publisher) { return new EventDatabase($publisher); }, ['publisher']); diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 0edffdf4dc..14b1fd356a 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -24,23 +24,12 @@ class Event public const FUNCTIONS_QUEUE_NAME = 'v1-functions'; public const FUNCTIONS_CLASS_NAME = 'FunctionsV1'; - /** remove */ - public const USAGE_QUEUE_NAME = 'v1-usage'; - public const USAGE_CLASS_NAME = 'UsageV1'; - - public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; - public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; - /** /remove */ - public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources'; public const STATS_RESOURCES_CLASS_NAME = 'StatsResourcesV1'; public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage'; public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1'; - public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump'; - public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1'; - public const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; public const WEBHOOK_CLASS_NAME = 'WebhooksV1'; diff --git a/src/Appwrite/Event/StatsUsageDump.php b/src/Appwrite/Event/StatsUsageDump.php deleted file mode 100644 index 0573a88040..0000000000 --- a/src/Appwrite/Event/StatsUsageDump.php +++ /dev/null @@ -1,44 +0,0 @@ -setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME) - ->setClass(Event::STATS_USAGE_DUMP_CLASS_NAME); - } - - /** - * Add Stats. - * - * @param array $stats - * @return self - */ - public function setStats(array $stats): self - { - $this->stats = $stats; - - return $this; - } - - /** - * Prepare the payload for the usage dump event. - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'stats' => $this->stats, - ]; - } -} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 4f4095aca4..eb544c140e 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -13,7 +13,6 @@ use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\StatsUsage; -use Appwrite\Platform\Workers\StatsUsageDump; use Appwrite\Platform\Workers\Webhooks; use Utopia\Platform\Service; @@ -32,7 +31,6 @@ class Workers extends Service ->addAction(Mails::getName(), new Mails()) ->addAction(Messaging::getName(), new Messaging()) ->addAction(Webhooks::getName(), new Webhooks()) - ->addAction(StatsUsageDump::getName(), new StatsUsageDump()) ->addAction(StatsUsage::getName(), new StatsUsage()) ->addAction(Migrations::getName(), new Migrations()) ->addAction(StatsResources::getName(), new StatsResources()) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php deleted file mode 100644 index 2832e97000..0000000000 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ /dev/null @@ -1,278 +0,0 @@ - true, - METRIC_BUCKETS => true, - METRIC_USERS => true, - METRIC_FUNCTIONS => true, - METRIC_TEAMS => true, - METRIC_MESSAGES => true, - METRIC_MAU => true, - METRIC_WEBHOOKS => true, - METRIC_PLATFORMS => true, - METRIC_PROVIDERS => true, - METRIC_TOPICS => true, - METRIC_KEYS => true, - METRIC_FILES => true, - METRIC_FILES_STORAGE => true, - METRIC_DEPLOYMENTS_STORAGE => true, - METRIC_BUILDS_STORAGE => true, - METRIC_DEPLOYMENTS => true, - METRIC_BUILDS => true, - METRIC_COLLECTIONS => true, - METRIC_DOCUMENTS => true, - METRIC_DATABASES_STORAGE => true, - ]; - - /** - * Skip metrics associated with parent IDs - * these need to be checked individually with `str_ends_with` - */ - protected array $skipParentIdMetrics = [ - '.files', - '.files.storage', - '.collections', - '.documents', - '.deployments', - '.deployments.storage', - '.builds', - '.builds.storage', - '.databases.storage' - ]; - - /** - * @var callable - */ - protected mixed $getLogsDB; - - protected array $periods = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; - - public static function getName(): string - { - return 'stats-usage-dump'; - } - - /** - * @throws \Exception - */ - public function __construct() - { - $this - ->inject('message') - ->inject('getProjectDB') - ->inject('getLogsDB') - ->inject('register') - ->callback([$this, 'action']); - } - - /** - * @param Message $message - * @param callable $getProjectDB - * @param callable $getLogsDB - * @param Registry $register - * @return void - * @throws Exception - * @throws \Throwable - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void - { - $this->getLogsDB = $getLogsDB; - $this->register = $register; - $payload = $message->getPayload() ?? []; - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - foreach ($payload['stats'] ?? [] as $stats) { - $project = new Document($stats['project'] ?? []); - $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? null; - if ($numberOfKeys === 0) { - continue; - } - - console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); - - try { - foreach ($stats['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - if (str_contains($key, METRIC_DATABASES_STORAGE)) { - // skip database storage calc as it's wrong and we plan to get this from StatsResources - continue; - } - - foreach ($this->periods as $period => $format) { - $time = null; - - if ($period !== 'inf') { - $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); - } - $id = \md5("{$time}_{$period}_{$key}"); - - $document = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - - - $this->projects[$project->getInternalId()]['project'] = new Document([ - '$id' => $project->getId(), - '$internalId' => $project->getInternalId(), - 'database' => $project->getAttribute('database'), - ]); - $this->projects[$project->getInternalId()]['stats'][] = $document; - - $this->prepareForLogsDB($project, $document); - } - } - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); - } - } - - $batchSize = $this->getBatchSize(); - $shouldProcessBatch = \count($this->projects) >= $batchSize; - if (!$shouldProcessBatch && \count($this->projects) > 0) { - $shouldProcessBatch = (\time() - $this->lastDispatchTime) >= self::BATCH_AGGREGATION_INTERVAL; - } - - if ($shouldProcessBatch || App::isDevelopment()) { - foreach ($this->projects as $internalId => $projectStats) { - if (empty($internalId)) { - continue; - } - try { - /** @var \Utopia\Database\Database $dbForProject */ - $dbForProject = $getProjectDB($projectStats['project']); - Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); - $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); - Console::success('Batch successfully written to DB'); - - unset($this->projects[$internalId]); - } catch (Throwable $e) { - Console::error('Error processing stats: ' . $e->getMessage()); - } - } - $this->lastDispatchTime = time(); - } - $this->writeToLogsDB(); - - } - - protected function prepareForLogsDB(Document $project, Document $stat) - { - if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { - return; - } - if (array_key_exists($stat->getAttribute('metric'), $this->skipBaseMetrics)) { - return; - } - foreach ($this->skipParentIdMetrics as $skipMetric) { - if (str_ends_with($stat->getAttribute('metric'), $skipMetric)) { - return; - } - } - $documentClone = new Document($stat->getArrayCopy()); - $documentClone->setAttribute('$tenant', (int) $project->getInternalId()); - $this->statDocuments[] = $documentClone; - } - - protected function writeToLogsDB(): void - { - if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { - Console::log('Dual Writing is disabled. Skipping...'); - return; - } - - $batchSize = $this->getBatchSize(); - $shouldProcessBatch = \count($this->statDocuments) >= $batchSize; - if (!$shouldProcessBatch && \count($this->statDocuments) > 0) { - $shouldProcessBatch = (\time() - $this->lastDispatchTimeLogsDB) >= self::BATCH_AGGREGATION_INTERVAL; - } - - if (!$shouldProcessBatch) { - return; - } - - /** @var \Utopia\Database\Database $dbForLogs*/ - $dbForLogs = call_user_func($this->getLogsDB); - $dbForLogs - ->setTenant(null) - ->setTenantPerDocument(true); - - try { - Console::log('Processing batch with ' . count($this->statDocuments) . ' stats'); - $dbForLogs->createOrUpdateDocumentsWithIncrease( - 'stats', - 'value', - $this->statDocuments - ); - Console::success('Usage logs pushed to Logs DB'); - } catch (Throwable $th) { - Console::error($th->getMessage()); - } finally { - $this->lastDispatchTimeLogsDB = time(); - } - - $this->register->get('pools')->get('logs')->reclaim(); - } -} diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 04b1408cd0..4b7062dc22 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -541,28 +541,4 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(503, $response['headers']['status-code']); } - - public function testStatsUsageDumpSuccess() - { - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertIsInt($response['body']['size']); - $this->assertLessThan(100, $response['body']['size']); - - /** - * Test for FAILURE - */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump?threshold=0', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), []); - $this->assertEquals(503, $response['headers']['status-code']); - } } From f4d01f4c37ef2ffdea8a8a8705c8e3ab2ce4d404 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 27 Mar 2025 08:36:52 +0000 Subject: [PATCH 06/29] fix: remove stats usage dump worker from docker --- Dockerfile | 1 - app/views/install/compose.phtml | 28 ---------------------------- bin/worker-stats-usage-dump | 3 --- docker-compose.yml | 32 -------------------------------- 4 files changed, 64 deletions(-) delete mode 100644 bin/worker-stats-usage-dump diff --git a/Dockerfile b/Dockerfile index 88d5ed030b..2f1ea6f279 100755 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-stats-usage && \ - chmod +x /usr/local/bin/worker-stats-usage-dump && \ chmod +x /usr/local/bin/stats-resources && \ chmod +x /usr/local/bin/worker-stats-resources diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 7dfe14fcef..96236ecd48 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -744,34 +744,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-worker-stats-usage-dump: - image: /: - entrypoint: worker-stats-usage-dump - <<: *x-logging - container_name: appwrite-worker-stats-usage-dump - restart: unless-stopped - networks: - - appwrite - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_USAGE_STATS - - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-task-scheduler-functions: image: /: entrypoint: schedule-functions diff --git a/bin/worker-stats-usage-dump b/bin/worker-stats-usage-dump deleted file mode 100644 index 98e3c2cac7..0000000000 --- a/bin/worker-stats-usage-dump +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php stats-usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 05ddba967a..7d89ec0783 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -823,38 +823,6 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-stats-usage-dump: - entrypoint: worker-stats-usage-dump - <<: *x-logging - container_name: appwrite-worker-stats-usage-dump - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_USAGE_STATS - - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL - - _APP_DATABASE_SHARED_TABLES - - _APP_STATS_USAGE_DUAL_WRITING_DBS - appwrite-task-scheduler-functions: entrypoint: schedule-functions <<: *x-logging From d36a09eec69c75e068af3f58292898e0c3013ba5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Mar 2025 05:54:48 +0000 Subject: [PATCH 07/29] remove database storage triggers --- app/controllers/api/databases.php | 13 ++----------- src/Appwrite/Platform/Workers/StatsUsageDump.php | 6 ------ 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0c37e1a765..4b9d3d18f2 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -534,7 +534,6 @@ App::post('/v1/databases') } $queueForEvents->setParam('databaseId', $database->getId()); - $queueForStatsUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -830,9 +829,6 @@ App::delete('/v1/databases/:databaseId') ->setParam('databaseId', $database->getId()) ->setPayload($response->output($database, Response::MODEL_DATABASE)); - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation - $response->noContent(); }); @@ -2732,9 +2728,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->setContext('database', $db) ->setPayload($response->output($attribute, $model)); - $queueForStatsUsage - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->noContent(); }); @@ -3356,8 +3349,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); // per collection $response->addHeader('X-Debug-Operations', $operations); @@ -4141,8 +4133,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForStatsUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); // per collection $response->addHeader('X-Debug-Operations', 1); diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 2832e97000..dd9ecd52d3 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -6,7 +6,6 @@ use Appwrite\Extend\Exception; use Throwable; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; @@ -151,11 +150,6 @@ class StatsUsageDump extends Action continue; } - if (str_contains($key, METRIC_DATABASES_STORAGE)) { - // skip database storage calc as it's wrong and we plan to get this from StatsResources - continue; - } - foreach ($this->periods as $period => $format) { $time = null; From 1389bab3d1a3d9d769c67b65f2bf2cf7b5f96551 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Mar 2025 05:56:14 +0000 Subject: [PATCH 08/29] remove db storage check --- src/Appwrite/Platform/Workers/StatsUsage.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 6439fa5fbc..a0c16449b6 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -347,11 +347,6 @@ class StatsUsage extends Action continue; } - if (str_contains($key, METRIC_DATABASES_STORAGE)) { - // skip database storage calc as it's wrong and we plan to get this from StatsResources - continue; - } - foreach ($this->periods as $period => $format) { $time = null; From d5ad714f5a211fc617c9751cd6975afeb883c06c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Mar 2025 06:09:03 +0000 Subject: [PATCH 09/29] fix missing queue name --- app/controllers/api/health.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 204ba0ad0a..81a77733ff 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -922,7 +922,6 @@ App::get('/v1/health/queue/failed/:name') Event::FUNCTIONS_QUEUE_NAME, Event::STATS_RESOURCES_QUEUE_NAME, Event::STATS_USAGE_QUEUE_NAME, - Event::STATS_USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_QUEUE_NAME, Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, From ec21effd2c466c4b06a96ec2a678adfaa97353a9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Mar 2025 03:55:06 +0000 Subject: [PATCH 10/29] Fix type error --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index a0c16449b6..ffd2a32648 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -332,7 +332,7 @@ class StatsUsage extends Action { foreach ($this->stats as $stats) { - $project = new Document($stats['project'] ?? []); + $project = $stats['project'] ?? new Document([]); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; $receivedAt = $stats['receivedAt'] ?? null; if ($numberOfKeys === 0) { From e14ae7ea4acc6f915d790ddda587c69b634ae5fc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Mar 2025 03:59:37 +0000 Subject: [PATCH 11/29] fix --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index ffd2a32648..71e1e1a9dd 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -323,7 +323,7 @@ class StatsUsage extends Action default: break; } - } catch (\Throwable $e) { + } catch (Throwable $e) { console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); } } From fd5a90075896a046e244d910fa844d0f3f34dd3a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 13 Apr 2025 03:46:59 +0000 Subject: [PATCH 12/29] update database --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 8db4706bb5..6fcbb16235 100644 --- a/composer.lock +++ b/composer.lock @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.64.1", + "version": "0.64.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b" + "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", - "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", + "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", "shasum": "" }, "require": { @@ -3547,9 +3547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.64.1" + "source": "https://github.com/utopia-php/database/tree/0.64.2" }, - "time": "2025-04-02T00:35:29+00:00" + "time": "2025-04-09T07:53:05+00:00" }, { "name": "utopia-php/domains", From ecdea8b487276f9c445884af1193df734205d079 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Apr 2025 18:07:12 +1200 Subject: [PATCH 13/29] Update database --- composer.json | 2 +- composer.lock | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/composer.json b/composer.json index 3920351c06..ab8aa56c17 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.64.*", + "utopia-php/database": "dev-feat-bulk-callbacks as 0.64.2", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 5cd5122694..8e27ee0e98 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6a54c8bc4f9f14cd3883f55880864630", + "content-hash": "b5844f87eec4cd045d791d0a8e54965e", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.64.2", + "version": "dev-feat-bulk-callbacks", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd" + "reference": "b6456ac11c4b00adea874f5dcba49f9904203563" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", - "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b6456ac11c4b00adea874f5dcba49f9904203563", + "reference": "b6456ac11c4b00adea874f5dcba49f9904203563", "shasum": "" }, "require": { @@ -3547,9 +3547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.64.2" + "source": "https://github.com/utopia-php/database/tree/feat-bulk-callbacks" }, - "time": "2025-04-09T07:53:05+00:00" + "time": "2025-04-14T05:53:57+00:00" }, { "name": "utopia-php/domains", @@ -8124,9 +8124,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-bulk-callbacks", + "alias": "0.64.2", + "alias_normalized": "0.64.2.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 9a8ba13dbf0983066c47603d733dd09c512327cf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Apr 2025 18:18:10 +1200 Subject: [PATCH 14/29] Fix syntax --- src/Appwrite/Platform/Tasks/Maintenance.php | 2 +- src/Appwrite/Platform/Tasks/StatsResources.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 62f504acf0..7c8f9bda78 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -50,7 +50,7 @@ class Maintenance extends Action $dbForPlatform->foreach( 'projects', [ - Query::equal('region', System::getEnv('_APP_REGION', 'default')) + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]) ], function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { $queueForDeletes diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index 88969ee2a4..ca2a6860ff 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -68,7 +68,7 @@ class StatsResources extends Action */ $this->foreachDocument($this->dbForPlatform, 'projects', [ Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours)), - Query::equal('region', System::getEnv('_APP_REGION', 'default')) + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]) ], function ($project) use ($queue) { $queue ->setProject($project) From 2a2bf7f154de7d9dca95e049890ef0c7848bd659 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Apr 2025 18:23:35 +1200 Subject: [PATCH 15/29] Fix maintenance --- src/Appwrite/Platform/Tasks/Maintenance.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 7c8f9bda78..2df75b22a8 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -49,9 +49,6 @@ class Maintenance extends Action $dbForPlatform->foreach( 'projects', - [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]) - ], function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { $queueForDeletes ->setType(DELETE_TYPE_MAINTENANCE) @@ -60,6 +57,7 @@ class Maintenance extends Action ->trigger(); }, [ + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::limit(100), ] ); From 498f9e40c8423927b24ddd4b2a46b403bf294ec3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Apr 2025 18:50:33 +1200 Subject: [PATCH 16/29] Update worker bulk method usage --- src/Appwrite/Platform/Workers/Databases.php | 15 ++++++--------- src/Appwrite/Platform/Workers/Deletes.php | 16 ++++++---------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 44738b557c..1b3311b756 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -563,22 +563,19 @@ class Databases extends Action $start = \microtime(true); try { - $documents = $database->deleteDocuments($collectionId, $queries); + $count = $database->deleteDocuments( + $collectionId, + $queries, + Database::DELETE_BATCH_SIZE, + $callback + ); } catch (\Throwable $th) { $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collectionId} {$tenant} :{$th->getMessage()}"); return; } - if (\is_callable($callback)) { - foreach ($documents as $document) { - $callback($document); - } - } - $end = \microtime(true); - $count = \count($documents); - Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds"); } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 5be865f42b..9d4e3e0f1e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1050,24 +1050,20 @@ class Deletes extends Action /** * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ - try { - $documents = $database->deleteDocuments($collection, $queries); + $count =$database->deleteDocuments( + $collection, + $queries, + Database::DELETE_BATCH_SIZE, + $callback + ); } catch (Throwable $th) { $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}"); return; } - if (\is_callable($callback)) { - foreach ($documents as $document) { - $callback($document); - } - } - $end = \microtime(true); - $count = \count($documents); - Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds"); } From 862664dd629221ec3126a1c6b0b2ad92622236c1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Apr 2025 19:05:42 +1200 Subject: [PATCH 17/29] Format --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9d4e3e0f1e..4a2cb60d32 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1051,7 +1051,7 @@ class Deletes extends Action * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ try { - $count =$database->deleteDocuments( + $count = $database->deleteDocuments( $collection, $queries, Database::DELETE_BATCH_SIZE, From 950cc420a2334348a8ec69f762abb011b17b4f00 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Apr 2025 19:40:51 +1200 Subject: [PATCH 18/29] Update database --- composer.json | 2 +- composer.lock | 39 +++++++++++++++------------------------ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/composer.json b/composer.json index ab8aa56c17..5b13227d2b 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-bulk-callbacks as 0.64.2", + "utopia-php/database": "0.65.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 8e27ee0e98..9800d5fc9d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b5844f87eec4cd045d791d0a8e54965e", + "content-hash": "51ff891ef6cee8a3f8c4e5187b7fd479", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-bulk-callbacks", + "version": "0.65.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "b6456ac11c4b00adea874f5dcba49f9904203563" + "reference": "e589efdc5da1216523a758e8af358866d4fb563f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/b6456ac11c4b00adea874f5dcba49f9904203563", - "reference": "b6456ac11c4b00adea874f5dcba49f9904203563", + "url": "https://api.github.com/repos/utopia-php/database/zipball/e589efdc5da1216523a758e8af358866d4fb563f", + "reference": "e589efdc5da1216523a758e8af358866d4fb563f", "shasum": "" }, "require": { @@ -3547,9 +3547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-bulk-callbacks" + "source": "https://github.com/utopia-php/database/tree/0.65.0" }, - "time": "2025-04-14T05:53:57+00:00" + "time": "2025-04-14T07:39:01+00:00" }, { "name": "utopia-php/domains", @@ -3660,16 +3660,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.4.0", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e" + "reference": "65095dac14037db0c822fb5e209e5bd3187a0303" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/46e791ff6a95864517750b9df6bbf4a17e3c9c4e", - "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/65095dac14037db0c822fb5e209e5bd3187a0303", + "reference": "65095dac14037db0c822fb5e209e5bd3187a0303", "shasum": "" }, "require": { @@ -3693,9 +3693,9 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.4.0" + "source": "https://github.com/utopia-php/fetch/tree/0.4.1" }, - "time": "2025-03-11T21:06:56+00:00" + "time": "2025-04-14T07:34:27+00:00" }, { "name": "utopia-php/framework", @@ -8124,18 +8124,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-bulk-callbacks", - "alias": "0.64.2", - "alias_normalized": "0.64.2.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From a46c5d10299462a03d70fe917ce59a31260df43a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Apr 2025 08:54:04 +0545 Subject: [PATCH 19/29] Update src/Appwrite/Platform/Workers/StatsUsage.php Co-authored-by: Jake Barnby --- src/Appwrite/Platform/Workers/StatsUsage.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 71e1e1a9dd..80aea081ad 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -330,7 +330,6 @@ class StatsUsage extends Action public function commitToDb(callable $getProjectDB): void { - foreach ($this->stats as $stats) { $project = $stats['project'] ?? new Document([]); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; From 93f86e57ea53fb124186706ee263a44fa36877c4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Apr 2025 03:15:24 +0000 Subject: [PATCH 20/29] use clone --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 80aea081ad..3e99ea972d 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -413,7 +413,7 @@ class StatsUsage extends Action return; } } - $documentClone = new Document($stat->getArrayCopy()); + $documentClone = clone $stat; $documentClone->setAttribute('$tenant', (int) $project->getInternalId()); $this->statDocuments[] = $documentClone; } From 4edec1494a76bf8067062344bb588700312ca8c6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Apr 2025 04:45:56 +0000 Subject: [PATCH 21/29] fix remove comments --- tests/e2e/General/UsageTest.php | 250 -------------------------------- 1 file changed, 250 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 2cceeeae05..cadc45fea2 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -6,7 +6,6 @@ use Appwrite\Functions\Specification; use Appwrite\Tests\Retry; use CURLFile; use DateTime; -use PHPUnit\Framework\ExpectationFailedException; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -593,255 +592,6 @@ class UsageTest extends Scope return $data; } - // public function testDatabaseStoragePrepare(): array - // { - // $response = $this->client->call( - // Client::METHOD_POST, - // '/databases', - // array_merge([ - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'databaseId' => 'unique()', - // 'name' => 'dbStorageStats', - // ] - // ); - - // $this->assertNotEmpty($response['body']['$id']); - // $databaseId = $response['body']['$id']; - - // $response = $this->client->call( - // Client::METHOD_POST, - // '/databases/' . $databaseId . '/collections', - // array_merge([ - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'collectionId' => 'unique()', - // 'name' => 'collectionStorageStats', - // 'documentSecurity' => false, - // 'permissions' => [ - // Permission::read(Role::any()), - // Permission::create(Role::any()), - // Permission::update(Role::any()), - // Permission::delete(Role::any()), - // ], - // ] - // ); - - // $this->assertNotEmpty($response['body']['$id']); - // $collectionId = $response['body']['$id']; - - // $response = $this->client->call( - // Client::METHOD_POST, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', - // array_merge([ - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'key' => 'data', - // 'size' => 100000, - // 'required' => true, - // ] - // ); - - // return [ - // 'databaseId' => $databaseId, - // 'collectionId' => $collectionId, - // ]; - // } - - // /** @depends testDatabaseStoragePrepare */ - // #[Retry(count: 1)] - // public function testDatabaseStorageStatsCreateDocument(array $data): array - // { - // $databaseId = $data['databaseId']; - // $collectionId = $data['collectionId']; - - // $originalProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $originalProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $originalProjectMetrics['body']); - - // $originalProjectMetrics = $originalProjectMetrics['body']; - - // $originalDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $originalDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $originalDatabaseMetrics['body']); - // $originalDatabaseMetrics = $originalDatabaseMetrics['body']; - - // // Create documents - // for ($i = 0; $i < 100; $i++) { - // $response = $this->client->call( - // Client::METHOD_POST, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', - // array_merge([ - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'documentId' => 'unique()', - // 'data' => ['data' => str_repeat('a', 10000)], - // ] - // ); - - // $this->assertEquals(201, $response['headers']['status-code']); - // } - - // for ($i = 0; $i < 3; $i++) { - // try { - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); - // $this->assertGreaterThan($originalProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); - - // $newProjectMetrics = $newProjectMetrics['body']; - - // $newDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); - // $this->assertGreaterThan($originalDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); - - // $newDatabaseMetrics = $newDatabaseMetrics['body']; - - // return [ - // 'databaseId' => $databaseId, - // 'collectionId' => $collectionId, - // 'currentProjectMetrics' => $newProjectMetrics, - // 'currentDatabaseMetrics' => $newDatabaseMetrics, - // ]; - // } catch (ExpectationFailedException $e) { - // if ($i === 2) { - // throw $e; - // } - // continue; - // } - // } - // } - - // /** @depends testDatabaseStorageStatsCreateDocument */ - // #[Retry(count: 1)] - // public function testDatabaseStorageStatsDeleteDocument(array $data): array - // { - // $databaseId = $data['databaseId']; - // $collectionId = $data['collectionId']; - // $currentProjectMetrics = $data['currentProjectMetrics']; - // $currentDatabaseMetrics = $data['currentDatabaseMetrics']; - - // $documents = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', - // array_merge([ - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'queries' => [ - // Query::limit(50)->toString() - // ] - // ] - // ); - - // foreach ($documents['body']['documents'] as $document) { - // $response = $this->client->call( - // Client::METHOD_DELETE, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document['$id'], - // array_merge([ - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()) - // ); - - // $this->assertEquals(204, $response['headers']['status-code']); - // } - - // for ($i = 0; $i < 3; $i++) { - // try { - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); - // $this->assertLessThan($currentProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); - - // $newProjectMetrics = $newProjectMetrics['body']; - - // $newDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); - // $this->assertLessThan($currentDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); - - // $newDatabaseMetrics = $newDatabaseMetrics['body']; - - // return [ - // 'databaseId' => $databaseId, - // 'collectionId' => $collectionId, - // 'currentProjectMetrics' => $newProjectMetrics, - // 'currentDatabaseMetrics' => $newDatabaseMetrics, - // ]; - // } catch (ExpectationFailedException $e) { - // if ($i === 2) { - // throw $e; - // } - // continue; - // } - // } - - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - // } - /** @depends testDatabaseStats */ public function testPrepareFunctionsStats(array $data): array { From f9141ca564adfcf8cb7967e54baaf2baa1c79b67 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Apr 2025 04:48:19 +0000 Subject: [PATCH 22/29] fix param types --- src/Appwrite/Platform/Workers/StatsUsage.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 3e99ea972d..66f285fcf5 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers; use Exception; use Throwable; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; @@ -85,7 +86,7 @@ class StatsUsage extends Action ]; /** - * @var callable + * @var callable(): Database */ protected mixed $getLogsDB; @@ -125,8 +126,8 @@ class StatsUsage extends Action /** * @param Message $message - * @param callable $getProjectDB - * @param callable $getLogsDB + * @param callable(): Database $getProjectDB + * @param callable(): Database $getLogsDB * @param Registry $register * @return void * @throws \Utopia\Database\Exception @@ -191,7 +192,7 @@ class StatsUsage extends Action * @param Document $project * @param Document $document * @param array $metrics - * @param callable $getProjectDB + * @param callable(): Database $getProjectDB * @return void */ private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void @@ -328,6 +329,11 @@ class StatsUsage extends Action } } + /** + * Commit stats to DB + * @param callable(): Database $getProjectDB + * @return void + */ public function commitToDb(callable $getProjectDB): void { foreach ($this->stats as $stats) { @@ -384,7 +390,6 @@ class StatsUsage extends Action continue; } try { - /** @var \Utopia\Database\Database $dbForProject */ $dbForProject = $getProjectDB($projectStats['project']); Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats'); $dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']); @@ -425,7 +430,6 @@ class StatsUsage extends Action return; } - /** @var \Utopia\Database\Database $dbForLogs*/ $dbForLogs = call_user_func($this->getLogsDB); $dbForLogs ->setTenant(null) From 49814c2a7e7fc1590df25f2ba1d017facdb79603 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Apr 2025 04:58:16 +0000 Subject: [PATCH 23/29] bring back stats usage dump --- .../Platform/Workers/StatsUsageDump.php | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 src/Appwrite/Platform/Workers/StatsUsageDump.php diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php new file mode 100644 index 0000000000..b9d486e0d8 --- /dev/null +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -0,0 +1,209 @@ + true, + METRIC_BUCKETS => true, + METRIC_USERS => true, + METRIC_FUNCTIONS => true, + METRIC_TEAMS => true, + METRIC_MESSAGES => true, + METRIC_MAU => true, + METRIC_WEBHOOKS => true, + METRIC_PLATFORMS => true, + METRIC_PROVIDERS => true, + METRIC_TOPICS => true, + METRIC_KEYS => true, + METRIC_FILES => true, + METRIC_FILES_STORAGE => true, + METRIC_DEPLOYMENTS_STORAGE => true, + METRIC_BUILDS_STORAGE => true, + METRIC_DEPLOYMENTS => true, + METRIC_BUILDS => true, + METRIC_COLLECTIONS => true, + METRIC_DOCUMENTS => true, + METRIC_DATABASES_STORAGE => true, + ]; + + /** + * Skip metrics associated with parent IDs + * these need to be checked individually with `str_ends_with` + */ + protected array $skipParentIdMetrics = [ + '.files', + '.files.storage', + '.collections', + '.documents', + '.deployments', + '.deployments.storage', + '.builds', + '.builds.storage', + '.databases.storage' + ]; + + /** + * @var callable + */ + protected mixed $getLogsDB; + + protected array $periods = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'stats-usage-dump'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + $this + ->inject('message') + ->inject('getProjectDB') + ->inject('getLogsDB') + ->inject('register') + ->callback([$this, 'action']); + } + + /** + * @param Message $message + * @param callable $getProjectDB + * @param callable $getLogsDB + * @param Registry $register + * @return void + * @throws Exception + * @throws \Throwable + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void + { + $this->getLogsDB = $getLogsDB; + $this->register = $register; + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + $receivedAt = $stats['receivedAt'] ?? null; + if ($numberOfKeys === 0) { + continue; + } + + console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + + try { + /** @var \Utopia\Database\Database $dbForProject */ + $dbForProject = $getProjectDB($project); + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + if (str_contains($key, METRIC_DATABASES_STORAGE)) { + continue; + } + + foreach ($this->periods as $period => $format) { + $time = null; + + if ($period !== 'inf') { + $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + } + $id = \md5("{$time}_{$period}_{$key}"); + + $document = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + + $documentClone = clone $document; + + $dbForProject->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + [$document] + ); + + $this->writeToLogsDB($project, $documentClone); + } + } + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); + } + } + } + + protected function writeToLogsDB(Document $project, Document $document): void + { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { + Console::log('Dual Writing is disabled. Skipping...'); + return; + } + + if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) { + return; + } + foreach ($this->skipParentIdMetrics as $skipMetric) { + if (str_ends_with($document->getAttribute('metric'), $skipMetric)) { + return; + } + } + + /** @var \Utopia\Database\Database $dbForLogs*/ + $dbForLogs = call_user_func($this->getLogsDB, $project); + + try { + $dbForLogs->createOrUpdateDocumentsWithIncrease( + 'stats', + 'value', + [$document] + ); + Console::success('Usage logs pushed to Logs DB'); + } catch (\Throwable $th) { + Console::error($th->getMessage()); + } + + $this->register->get('pools')->get('logs')->reclaim(); + } +} From d0181787b916ea843da81e1cc3a8f8957080ebe2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 15 Apr 2025 17:38:34 +0400 Subject: [PATCH 24/29] fix: usage test assertion --- tests/e2e/General/UsageTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index e614e2e185..9406e6af35 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -148,7 +148,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(31, count($response['body'])); + $this->assertGreaterThanOrEqual(31, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); From 304f67c88f1421a454e1a74e88d58dabc6e59626 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 15 Apr 2025 18:18:40 +0400 Subject: [PATCH 25/29] fix: usage test assertion --- tests/e2e/General/UsageTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 9406e6af35..d0e37e5639 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -327,7 +327,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(31, count($response['body'])); + $this->assertGreaterThanOrEqual(31, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -548,7 +548,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(31, count($response['body'])); + $this->assertGreaterThanOrEqual(31, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); From ba58a372d17a0b88f80cbcdd99006bf88bb260b5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 16 Apr 2025 10:50:53 +1200 Subject: [PATCH 26/29] Throw on key not found --- app/controllers/shared/api.php | 48 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index bf35f9073b..22dc54d2c9 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -238,34 +238,36 @@ App::init() subject: 'keys' ); - if ($dbKey) { - $accessedAt = $dbKey->getAttribute('accessedAt', ''); + if (!$dbKey) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { - $dbKey->setAttribute('accessedAt', DateTime::now()); + $accessedAt = $dbKey->getAttribute('accessedAt', ''); + + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { + $dbKey->setAttribute('accessedAt', DateTime::now()); + $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); + $dbForPlatform->purgeCachedDocument('projects', $project->getId()); + } + + $sdkValidator = new WhiteList($servers, true); + $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); + + if ($sdkValidator->isValid($sdk)) { + $sdks = $dbKey->getAttribute('sdks', []); + + if (!in_array($sdk, $sdks)) { + $sdks[] = $sdk; + $dbKey->setAttribute('sdks', $sdks); + + /** Update access time as well */ + $dbKey->setAttribute('accessedAt', Datetime::now()); $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); } - - $sdkValidator = new WhiteList($servers, true); - $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); - - if ($sdkValidator->isValid($sdk)) { - $sdks = $dbKey->getAttribute('sdks', []); - - if (!in_array($sdk, $sdks)) { - $sdks[] = $sdk; - $dbKey->setAttribute('sdks', $sdks); - - /** Update access time as well */ - $dbKey->setAttribute('accessedAt', Datetime::now()); - $dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey); - $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - } - } - - $queueForAudits->setUser($user); } + + $queueForAudits->setUser($user); } } // Admin User Authentication elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) { From bea723ff2ef3e02e3d48a24c4d8c7c41d1756e1f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 16 Apr 2025 12:04:19 +1200 Subject: [PATCH 27/29] Add test --- composer.lock | 34 +++++++++---------- .../Projects/ProjectsConsoleClientTest.php | 34 +++++++++++++++++++ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/composer.lock b/composer.lock index c6c9ae2900..c8c1e3c192 100644 --- a/composer.lock +++ b/composer.lock @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.3", + "version": "1.2.4", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc" + "reference": "47fcb66ae5328c5a799195247b1dce551d85873e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/0e7804c176c4b09d95b7985400aa38ce544cb7fc", - "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/47fcb66ae5328c5a799195247b1dce551d85873e", + "reference": "47fcb66ae5328c5a799195247b1dce551d85873e", "shasum": "" }, "require": { @@ -1451,7 +1451,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-04-08T09:55:41+00:00" + "time": "2025-04-15T07:02:07+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3351,16 +3351,16 @@ }, { "name": "utopia-php/cli", - "version": "0.15.1", + "version": "0.15.2", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65" + "reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/d69bbe51a6a94dc4e5bcdd542b5938038b985a65", - "reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/da00ff6b8b29a826a1794002ae43442cdf3a0f5f", + "reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f", "shasum": "" }, "require": { @@ -3394,9 +3394,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.15.1" + "source": "https://github.com/utopia-php/cli/tree/0.15.2" }, - "time": "2024-10-04T13:55:36+00:00" + "time": "2025-04-15T10:08:48+00:00" }, { "name": "utopia-php/compression", @@ -4767,16 +4767,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.11", + "version": "0.40.12", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" + "reference": "182ec17848f81b78c336379bac94ff92b7a73365" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", - "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/182ec17848f81b78c336379bac94ff92b7a73365", + "reference": "182ec17848f81b78c336379bac94ff92b7a73365", "shasum": "" }, "require": { @@ -4812,9 +4812,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.12" }, - "time": "2025-03-26T10:53:16+00:00" + "time": "2025-04-02T23:36:11+00:00" }, { "name": "doctrine/annotations", diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 8e2ab03880..a53d53ace4 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -2989,6 +2989,40 @@ class ProjectsConsoleClientTest extends Scope $this->assertEmpty($response['body']); } + /** + * @depends testCreateProject + */ + public function testUseInvalidKey(array $data): void + { + $id = $data['projectId'] ?? ''; + + // Create bucket + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $id, + 'x-appwrite-mode' => 'admin', + ], $this->getHeaders()), [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'permission' => 'file', + 'enabled' => true, + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + $this->assertNotEmpty($bucket['body']['$id']); + + $bucketId = $bucket['body']['$id']; + + // Use invalid key to read + $response = $this->client->call(Client::METHOD_GET, "/storage/buckets/{$bucketId}/files", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $id, + 'x-appwrite-key' => 'invalid-key' + ]); + + $this->assertEquals(401, $response['headers']['status-code']); + } + // JWT Keys /** From 53660f310019859fd4c5e9d60cf35627e8d4fe7a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 16 Apr 2025 12:30:22 +1200 Subject: [PATCH 28/29] Add more cases --- .../Projects/ProjectsConsoleClientTest.php | 142 ++++++++++++------ 1 file changed, 93 insertions(+), 49 deletions(-) diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index a53d53ace4..f89428d79a 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -2786,32 +2786,35 @@ class ProjectsConsoleClientTest extends Scope */ public function testValidateProjectKey($data): void { - $id = $data['projectId'] ?? ''; + $projectId = $data['projectId'] ?? ''; + $teamId = $data['teamId'] ?? ''; /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_POST, '/projects/' . $id . '/keys', array_merge([ + + // Expiring key + $response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'name' => 'Key Test', - 'scopes' => ['health.read'], + 'scopes' => ['users.write'], 'expire' => DateTime::addSeconds(new \DateTime(), 3600), ]); - $response = $this->client->call(Client::METHOD_GET, '/health', [ + $response = $this->client->call(Client::METHOD_POST, '/users', [ 'content-type' => 'application/json', - 'x-appwrite-project' => $id, + 'x-appwrite-project' => $projectId, 'x-appwrite-key' => $response['body']['secret'] - ], []); + ], [ + 'userId' => ID::unique(), + ]); - $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(201, $response['headers']['status-code']); - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_POST, '/projects/' . $id . '/keys', array_merge([ + // No expiry + $response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -2822,7 +2825,7 @@ class ProjectsConsoleClientTest extends Scope $response = $this->client->call(Client::METHOD_GET, '/health', [ 'content-type' => 'application/json', - 'x-appwrite-project' => $id, + 'x-appwrite-project' => $projectId, 'x-appwrite-key' => $response['body']['secret'] ], []); @@ -2831,7 +2834,9 @@ class ProjectsConsoleClientTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_POST, '/projects/' . $id . '/keys', array_merge([ + + // Expired key + $response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -2842,9 +2847,82 @@ class ProjectsConsoleClientTest extends Scope $response = $this->client->call(Client::METHOD_GET, '/health', [ 'content-type' => 'application/json', - 'x-appwrite-project' => $id, + 'x-appwrite-project' => $projectId, 'x-appwrite-key' => $response['body']['secret'] - ], []); + ]); + + $this->assertEquals(401, $response['headers']['status-code']); + + // Invalid key + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-mode' => 'admin', + ], $this->getHeaders()), [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + $this->assertNotEmpty($bucket['body']['$id']); + + $bucketId = $bucket['body']['$id']; + + $response = $this->client->call(Client::METHOD_GET, "/storage/buckets/{$bucketId}/files", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => 'invalid-key' + ]); + + $this->assertEquals(401, $response['headers']['status-code']); + + // Invalid scopes + $response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['teams.read'], + 'expire' => DateTime::addSeconds(new \DateTime(), 3600), + ]); + + $response = $this->client->call(Client::METHOD_GET, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $response['body']['secret'] + ]); + + $this->assertEquals(401, $response['headers']['status-code']); + + // Invalid key from different project + $response = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Project Test 2', + 'teamId' => $teamId, + 'region' => System::getEnv('_APP_REGION', 'default') + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + $project2Id = $response['body']['$id']; + + $response = $this->client->call(Client::METHOD_POST, '/projects/' . $project2Id . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['health.read'], + 'expire' => DateTime::addSeconds(new \DateTime(), 3600), + ]); + + $response = $this->client->call(Client::METHOD_GET, '/health', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $response['body']['secret'] + ]); $this->assertEquals(401, $response['headers']['status-code']); } @@ -2989,40 +3067,6 @@ class ProjectsConsoleClientTest extends Scope $this->assertEmpty($response['body']); } - /** - * @depends testCreateProject - */ - public function testUseInvalidKey(array $data): void - { - $id = $data['projectId'] ?? ''; - - // Create bucket - $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $id, - 'x-appwrite-mode' => 'admin', - ], $this->getHeaders()), [ - 'bucketId' => ID::unique(), - 'name' => 'Test Bucket', - 'permission' => 'file', - 'enabled' => true, - ]); - - $this->assertEquals(201, $bucket['headers']['status-code']); - $this->assertNotEmpty($bucket['body']['$id']); - - $bucketId = $bucket['body']['$id']; - - // Use invalid key to read - $response = $this->client->call(Client::METHOD_GET, "/storage/buckets/{$bucketId}/files", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $id, - 'x-appwrite-key' => 'invalid-key' - ]); - - $this->assertEquals(401, $response['headers']['status-code']); - } - // JWT Keys /** From b9e8c1d8bce09480d259bba6586e9e2a9cca40d4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 16 Apr 2025 13:07:11 +1200 Subject: [PATCH 29/29] Skip server console test --- tests/e2e/Services/Databases/DatabasesBase.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 0f57f94515..dca832ac01 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -90,6 +90,12 @@ trait DatabasesBase */ public function testConsoleProject(array $data) { + if ($this->getSide() === 'server') { + // Server side can't get past the invalid key check anyway + $this->expectNotToPerformAssertions(); + return; + } + $response = $this->client->call( Client::METHOD_GET, '/databases/console/collections/' . $data['moviesId'] . '/documents',