From d533113d867b09e22083de09336c8704702658d0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 01:44:33 +1300 Subject: [PATCH 01/17] Update usage dump to upsert documents in batches --- src/Appwrite/Platform/Workers/UsageDump.php | 208 +++++++++++--------- 1 file changed, 113 insertions(+), 95 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 2f1d13f29a..1e3d3149ce 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -8,6 +8,7 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; +use Utopia\Database\Exception\NotFound; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; @@ -70,10 +71,11 @@ class UsageDump extends Action continue; } - console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); try { $dbForProject = $getProjectDB($project); + foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; @@ -83,109 +85,73 @@ class UsageDump extends Action try { $this->handleDatabaseStorage($key, $dbForProject); } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); + Console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); } continue; } + $documents = []; + foreach ($this->periods as $period => $format) { $time = 'inf' === $period ? null : date($format, time()); $id = \md5("{$time}_{$period}_{$key}"); - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } + $documents[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); } + + $dbForProject->createOrUpdateDocumentsWithInplaceIncrease( + collection: 'stats', + attribute: 'value', + documents: $documents + ); } } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); + Console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); } } } private function handleDatabaseStorage(string $key, Database $dbForProject): void { - $data = explode('.', $key); - $start = microtime(true); + $data = \explode('.', $key); + $start = \microtime(true); - $updateMetric = function (Database $dbForProject, int $value, string $key, string $period, string|null $time) { - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } - }; + $documents = []; foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); + $time = 'inf' === $period ? null : \date($format, \time()); $id = \md5("{$time}_{$period}_{$key}"); $value = 0; $previousValue = 0; + try { - $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); - } catch (\Exception $e) { + $previousValue = $dbForProject + ->getDocument('stats', $id) + ->getAttribute('value', 0); + } catch (\Exception) { // No previous value } - switch (count($data)) { + switch (\count($data)) { // Collection Level case METRIC_COLLECTION_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; $collectionInternalId = $data[1]; try { $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { + if (!$e instanceof NotFound) { throw $e; } } @@ -197,18 +163,43 @@ class UsageDump extends Action break; } - // Update Collection - $updateMetric($dbForProject, $diff, $key, $period, $time); + $databaseKey = \str_replace( + ['{databaseInternalId}'], + [$data[0]], + METRIC_DATABASE_ID_STORAGE + ); - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + // Database + $documents[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $databaseKey, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + // Collection + $documents[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + + // Project + $documents[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => METRIC_DATABASES_STORAGE, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); break; - // Database Level + // Database Level case METRIC_DATABASE_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); $databaseInternalId = $data[0]; @@ -217,8 +208,7 @@ class UsageDump extends Action try { $collections = $dbForProject->find('database_' . $databaseInternalId); } catch (\Exception $e) { - // Database not found - if ($e->getMessage() !== 'Collection not found') { + if (!$e instanceof NotFound) { throw $e; } } @@ -227,8 +217,7 @@ class UsageDump extends Action try { $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { + if (!$e instanceof NotFound) { throw $e; } } @@ -240,19 +229,37 @@ class UsageDump extends Action break; } - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + // Database + $databaseKey = str_replace( + ['{databaseInternalId}'], + [$data[0]], + METRIC_DATABASE_ID_STORAGE + ); - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + $documents[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $databaseKey, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); + + // Project + $documents[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => METRIC_DATABASES_STORAGE, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); break; - // Project Level + // Project Level case METRIC_PROJECT_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); - // Get all project databases - $databases = $dbForProject->find('database'); + + $databases = $dbForProject->find('databases'); // Recalculate all databases foreach ($databases as $database) { @@ -262,8 +269,7 @@ class UsageDump extends Action try { $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { + if (!$e instanceof NotFound) { throw $e; } } @@ -272,15 +278,27 @@ class UsageDump extends Action $diff = $value - $previousValue; - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + // Project + $documents[] = new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => METRIC_DATABASES_STORAGE, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); break; } } + $dbForProject->createOrUpdateDocumentsWithInplaceIncrease( + collection: 'stats', + attribute: 'value', + documents: $documents + ); + $end = microtime(true); - console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); + Console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); } } From e0eea5b13d9601371bbcec1f2ab33cd3f1a334f6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 01:47:06 +1300 Subject: [PATCH 02/17] Update db --- composer.json | 2 +- composer.lock | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/composer.json b/composer.json index 9674bce2fd..1a641c06ce 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.56.4", + "utopia-php/database": "dev-feat-batch-upsert as 0.56.4", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index cd14e512a7..4d6e62c2ff 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3cd37ea04612e04b9e76eb51c5da41dd", + "content-hash": "8b621344d83a71a93913bfc12d1ebf4f", "packages": [ { "name": "adhocore/jwt", @@ -3476,16 +3476,16 @@ }, { "name": "utopia-php/database", - "version": "0.56.4", + "version": "dev-feat-batch-upsert", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "240478a60797124a885ceac40046fe47c22415b7" + "reference": "c9170c85c07997fc3718f35928b252062a02070a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/240478a60797124a885ceac40046fe47c22415b7", - "reference": "240478a60797124a885ceac40046fe47c22415b7", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c9170c85c07997fc3718f35928b252062a02070a", + "reference": "c9170c85c07997fc3718f35928b252062a02070a", "shasum": "" }, "require": { @@ -3526,9 +3526,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.56.4" + "source": "https://github.com/utopia-php/database/tree/feat-batch-upsert" }, - "time": "2025-01-20T09:22:08+00:00" + "time": "2025-01-22T12:38:29+00:00" }, { "name": "utopia-php/domains", @@ -8554,9 +8554,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-batch-upsert", + "alias": "0.56.4", + "alias_normalized": "0.56.4.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 343ef4944bb61a92e803faa937831b03da7479ba Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 01:57:23 +1300 Subject: [PATCH 03/17] Format --- composer.lock | 8 ++++---- src/Appwrite/Platform/Workers/UsageDump.php | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index 4d6e62c2ff..dac940b775 100644 --- a/composer.lock +++ b/composer.lock @@ -3480,12 +3480,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "c9170c85c07997fc3718f35928b252062a02070a" + "reference": "fb3fc216f3c8006528f42264216067aceb2f392c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/c9170c85c07997fc3718f35928b252062a02070a", - "reference": "c9170c85c07997fc3718f35928b252062a02070a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/fb3fc216f3c8006528f42264216067aceb2f392c", + "reference": "fb3fc216f3c8006528f42264216067aceb2f392c", "shasum": "" }, "require": { @@ -3528,7 +3528,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/feat-batch-upsert" }, - "time": "2025-01-22T12:38:29+00:00" + "time": "2025-01-22T12:49:21+00:00" }, { "name": "utopia-php/domains", diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 1e3d3149ce..87faf9263c 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -7,7 +7,6 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception\Duplicate; use Utopia\Database\Exception\NotFound; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -199,7 +198,7 @@ class UsageDump extends Action 'region' => System::getEnv('_APP_REGION', 'default'), ]); break; - // Database Level + // Database Level case METRIC_DATABASE_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); $databaseInternalId = $data[0]; @@ -255,7 +254,7 @@ class UsageDump extends Action 'region' => System::getEnv('_APP_REGION', 'default'), ]); break; - // Project Level + // Project Level case METRIC_PROJECT_LEVEL_STORAGE: Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); From 5e9955ed3e09da032f9030d43ad85e338d7270ca Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 17:42:12 +1300 Subject: [PATCH 04/17] Update database --- composer.json | 6 ++-- composer.lock | 95 +++++++++++++++++++++++---------------------------- 2 files changed, 46 insertions(+), 55 deletions(-) diff --git a/composer.json b/composer.json index 1a641c06ce..e024cff1c6 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.47.*", + "utopia-php/abuse": "0.48.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.47.*", + "utopia-php/audit": "0.48.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-batch-upsert as 0.56.4", + "utopia-php/database": "0.57.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index dac940b775..23badedd08 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8b621344d83a71a93913bfc12d1ebf4f", + "content-hash": "8927ec7d3cfa460ce223e4c13cf61ada", "packages": [ { "name": "adhocore/jwt", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.47.0", + "version": "0.48.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "2b52bb362234d4072b647ed57db1b3be030f57c2" + "reference": "8387c65cc7148af58adbbede06eedc1a7b568e57" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/2b52bb362234d4072b647ed57db1b3be030f57c2", - "reference": "2b52bb362234d4072b647ed57db1b3be030f57c2", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/8387c65cc7148af58adbbede06eedc1a7b568e57", + "reference": "8387c65cc7148af58adbbede06eedc1a7b568e57", "shasum": "" }, "require": { @@ -3153,13 +3153,13 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.56.*" + "utopia-php/database": "0.57.*" }, "require-dev": { - "laravel/pint": "1.5.*", - "phpbench/phpbench": "^1.2", - "phpstan/phpstan": "^1.9", - "phpunit/phpunit": "^9.4" + "laravel/pint": "1.*", + "phpbench/phpbench": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "9.*" }, "type": "library", "autoload": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.47.0" + "source": "https://github.com/utopia-php/abuse/tree/0.48.0" }, - "time": "2025-01-15T02:41:02+00:00" + "time": "2025-01-23T04:40:14+00:00" }, { "name": "utopia-php/analytics", @@ -3233,26 +3233,26 @@ }, { "name": "utopia-php/audit", - "version": "0.47.0", + "version": "0.48.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "1ebd5784ba68645073426f2f04a67726a1bde4d7" + "reference": "6aab185fce3ba7878b0f26cc8b4eefa1663fb395" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/1ebd5784ba68645073426f2f04a67726a1bde4d7", - "reference": "1ebd5784ba68645073426f2f04a67726a1bde4d7", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/6aab185fce3ba7878b0f26cc8b4eefa1663fb395", + "reference": "6aab185fce3ba7878b0f26cc8b4eefa1663fb395", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.56.*" + "utopia-php/database": "0.57.*" }, "require-dev": { - "laravel/pint": "1.5.*", - "phpstan/phpstan": "^1.8", - "phpunit/phpunit": "^9.3" + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "9.*" }, "type": "library", "autoload": { @@ -3274,9 +3274,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.47.0" + "source": "https://github.com/utopia-php/audit/tree/0.48.0" }, - "time": "2025-01-15T02:40:53+00:00" + "time": "2025-01-23T04:40:07+00:00" }, { "name": "utopia-php/cache", @@ -3476,16 +3476,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-batch-upsert", + "version": "0.57.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "fb3fc216f3c8006528f42264216067aceb2f392c" + "reference": "6ec27ade27e1b42b79ac42b06869e9c34d529357" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/fb3fc216f3c8006528f42264216067aceb2f392c", - "reference": "fb3fc216f3c8006528f42264216067aceb2f392c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/6ec27ade27e1b42b79ac42b06869e9c34d529357", + "reference": "6ec27ade27e1b42b79ac42b06869e9c34d529357", "shasum": "" }, "require": { @@ -3526,9 +3526,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-batch-upsert" + "source": "https://github.com/utopia-php/database/tree/0.57.1" }, - "time": "2025-01-22T12:49:21+00:00" + "time": "2025-01-23T04:26:35+00:00" }, { "name": "utopia-php/domains", @@ -3929,35 +3929,35 @@ }, { "name": "utopia-php/migration", - "version": "0.6.15", + "version": "0.6.16", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "e849ec3e7ad38f5f5273ebb0132b112639cdf01c" + "reference": "a1da9b75a0e406ea8caca0d61b57a4d206ea0715" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/e849ec3e7ad38f5f5273ebb0132b112639cdf01c", - "reference": "e849ec3e7ad38f5f5273ebb0132b112639cdf01c", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/a1da9b75a0e406ea8caca0d61b57a4d206ea0715", + "reference": "a1da9b75a0e406ea8caca0d61b57a4d206ea0715", "shasum": "" }, "require": { - "appwrite/appwrite": "11.1.*", + "appwrite/appwrite": "11.*", "ext-curl": "*", "ext-openssl": "*", - "php": "8.3.*", - "utopia-php/database": "0.56.*", + "php": ">=8.3", + "utopia-php/database": "0.57.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" }, "require-dev": { "ext-pdo": "*", - "laravel/pint": "1.17.*", - "phpstan/phpstan": "1.11.*", - "phpunit/phpunit": "11.2.*", + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "11.*", "utopia-php/cli": "0.16.*", - "vlucas/phpdotenv": "5.6.*" + "vlucas/phpdotenv": "5.*" }, "type": "library", "autoload": { @@ -3979,9 +3979,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.15" + "source": "https://github.com/utopia-php/migration/tree/0.6.16" }, - "time": "2025-01-15T04:55:08+00:00" + "time": "2025-01-23T04:34:02+00:00" }, { "name": "utopia-php/mongo", @@ -8554,18 +8554,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-batch-upsert", - "alias": "0.56.4", - "alias_normalized": "0.56.4.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 2a03bbf5b8a805c2cc15311a92fffe8ffcdd0435 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 18:20:00 +1300 Subject: [PATCH 05/17] Update database --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 23badedd08..02c3b65284 100644 --- a/composer.lock +++ b/composer.lock @@ -3476,16 +3476,16 @@ }, { "name": "utopia-php/database", - "version": "0.57.1", + "version": "0.57.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "6ec27ade27e1b42b79ac42b06869e9c34d529357" + "reference": "bd6f080dd9f4210349a6a862fa6da65a4d9d6339" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/6ec27ade27e1b42b79ac42b06869e9c34d529357", - "reference": "6ec27ade27e1b42b79ac42b06869e9c34d529357", + "url": "https://api.github.com/repos/utopia-php/database/zipball/bd6f080dd9f4210349a6a862fa6da65a4d9d6339", + "reference": "bd6f080dd9f4210349a6a862fa6da65a4d9d6339", "shasum": "" }, "require": { @@ -3526,9 +3526,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.57.1" + "source": "https://github.com/utopia-php/database/tree/0.57.2" }, - "time": "2025-01-23T04:26:35+00:00" + "time": "2025-01-23T05:19:02+00:00" }, { "name": "utopia-php/domains", From 65c2b99dfe9ba4c8f07a1b0f9f5d5cfb3853a2b8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 18:32:36 +1300 Subject: [PATCH 06/17] Fix method refs --- src/Appwrite/Platform/Workers/UsageDump.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 87faf9263c..99e312c64e 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -105,7 +105,7 @@ class UsageDump extends Action ]); } - $dbForProject->createOrUpdateDocumentsWithInplaceIncrease( + $dbForProject->createOrUpdateDocumentsWithIncrease( collection: 'stats', attribute: 'value', documents: $documents @@ -290,7 +290,7 @@ class UsageDump extends Action } } - $dbForProject->createOrUpdateDocumentsWithInplaceIncrease( + $dbForProject->createOrUpdateDocumentsWithIncrease( collection: 'stats', attribute: 'value', documents: $documents From 8cd400075143126313e1302e95ab0c26f1b9ee59 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 23:25:10 +1300 Subject: [PATCH 07/17] One write per project --- src/Appwrite/Platform/Workers/UsageDump.php | 350 ++++++++++---------- 1 file changed, 172 insertions(+), 178 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 99e312c64e..3590398aaa 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -38,9 +38,7 @@ class UsageDump extends Action $this ->inject('message') ->inject('getProjectDB') - ->callback(function (Message $message, callable $getProjectDB) { - $this->action($message, $getProjectDB); - }); + ->callback([$this, 'action']); } /** @@ -57,45 +55,38 @@ class UsageDump extends Action throw new Exception('Missing payload'); } - - foreach ($payload['stats'] ?? [] as $stats) { - $project = new Document($stats['project'] ?? []); - - /** - * End temp bug fallback - */ - $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? 'NONE'; - if ($numberOfKeys === 0) { - continue; - } - - Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); - - try { + try { + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); $dbForProject = $getProjectDB($project); + $projectDocuments = []; + $databaseCache = []; + $collectionSizeCache = []; foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; } - if (str_contains($key, METRIC_DATABASES_STORAGE)) { - try { - $this->handleDatabaseStorage($key, $dbForProject); - } catch (\Exception $e) { - Console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); - } - continue; - } - - $documents = []; - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); + $time = 'inf' === $period ? null : \date($format, \time()); $id = \md5("{$time}_{$period}_{$key}"); - $documents[] = new Document([ + if (\str_contains($key, METRIC_DATABASES_STORAGE)) { + $this->handleDatabaseStorage( + $id, + $key, + $time, + $period, + $dbForProject, + $projectDocuments, + $databaseCache, + $collectionSizeCache + ); + continue; + } + + $projectDocuments[] = new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -104,200 +95,203 @@ class UsageDump extends Action 'region' => System::getEnv('_APP_REGION', 'default'), ]); } - - $dbForProject->createOrUpdateDocumentsWithIncrease( - collection: 'stats', - attribute: 'value', - documents: $documents - ); } - } catch (\Exception $e) { - Console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); + + $dbForProject->createOrUpdateDocumentsWithIncrease( + collection: 'stats', + attribute: 'value', + documents: $projectDocuments + ); } + } catch (\Exception $e) { + Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage()); } } - private function handleDatabaseStorage(string $key, Database $dbForProject): void + private function handleDatabaseStorage( + string $id, + string $key, + string $time, + string $period, + Database $dbForProject, + array &$projectDocuments, + array &$databaseCache, + array &$collectionSizeCache, + ): void { $data = \explode('.', $key); - $start = \microtime(true); + $value = 0; + $previousValue = 0; - $documents = []; + try { + $previousValue = $dbForProject + ->getDocument('stats', $id) + ->getAttribute('value', 0); + } catch (\Exception) { + // No previous value + } - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : \date($format, \time()); - $id = \md5("{$time}_{$period}_{$key}"); + switch (\count($data)) { + case METRIC_COLLECTION_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - $value = 0; - $previousValue = 0; - - try { - $previousValue = $dbForProject - ->getDocument('stats', $id) - ->getAttribute('value', 0); - } catch (\Exception) { - // No previous value - } - - switch (\count($data)) { - // Collection Level - case METRIC_COLLECTION_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; + $databaseInternalId = $data[0]; + $collectionInternalId = $data[1]; + $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}"; + if (!isset($collectionSizeCache[$collectionId])) { try { - $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); + $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); } catch (\Exception $e) { if (!$e instanceof NotFound) { throw $e; } + $collectionSizeCache[$collectionId] = 0; } + } - // Compare with previous value - $diff = $value - $previousValue; + $value = $collectionSizeCache[$collectionId]; - if ($diff === 0) { - break; - } - - $databaseKey = \str_replace( - ['{databaseInternalId}'], - [$data[0]], - METRIC_DATABASE_ID_STORAGE - ); - - // Database - $documents[] = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $databaseKey, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - - // Collection - $documents[] = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - - // Project - $documents[] = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => METRIC_DATABASES_STORAGE, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); + $diff = $value - $previousValue; + if ($diff === 0) { break; - // Database Level - case METRIC_DATABASE_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; + } - $collections = []; + $keys = [ + $key, + \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), + METRIC_DATABASES_STORAGE + ]; + + foreach ($keys as $metric) { + $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); + } + break; + + case METRIC_DATABASE_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; + $databaseId = "database_{$databaseInternalId}"; + + if (!isset($databaseCache[$databaseId])) { try { - $collections = $dbForProject->find('database_' . $databaseInternalId); + $databaseCache[$databaseId] = $dbForProject->find($databaseId); } catch (\Exception $e) { if (!$e instanceof NotFound) { throw $e; } + $databaseCache[$databaseId] = []; } + } - foreach ($collections as $collection) { + foreach ($databaseCache[$databaseId] as $collection) { + $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; + + if (!isset($collectionSizeCache[$collectionId])) { try { - $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); + $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); } catch (\Exception $e) { if (!$e instanceof NotFound) { throw $e; } + $collectionSizeCache[$collectionId] = 0; + } + } + $value += $collectionSizeCache[$collectionId]; + } + + $diff = $value - $previousValue; + if ($diff === 0) { + break; + } + + $keys = [ + \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), + METRIC_DATABASES_STORAGE + ]; + + foreach ($keys as $metric) { + $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); + } + break; + + case METRIC_PROJECT_LEVEL_STORAGE: + if (!isset($databaseCache['*'])) { + try { + $databaseCache['*'] = $dbForProject->find('databases'); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $databaseCache['*'] = []; + } + } + + foreach ($databaseCache['*'] as $database) { + $databaseId = "database_{$database->getInternalId()}"; + if (!isset($databaseCache[$databaseId])) { + try { + $databaseCache[$databaseId] = $dbForProject->find($databaseId); + } catch (\Exception $e) { + if (!$e instanceof NotFound) { + throw $e; + } + $databaseCache[$databaseId] = []; } } - $diff = $value - $previousValue; + foreach ($databaseCache[$databaseId] as $collection) { + $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; - if ($diff === 0) { - break; - } - - // Database - $databaseKey = str_replace( - ['{databaseInternalId}'], - [$data[0]], - METRIC_DATABASE_ID_STORAGE - ); - - $documents[] = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $databaseKey, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - - // Project - $documents[] = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => METRIC_DATABASES_STORAGE, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); - break; - // Project Level - case METRIC_PROJECT_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); - - $databases = $dbForProject->find('databases'); - - // Recalculate all databases - foreach ($databases as $database) { - $collections = $dbForProject->find('database_' . $database->getInternalId()); - - foreach ($collections as $collection) { + if (!isset($collectionSizeCache[$collectionId])) { try { - $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId); } catch (\Exception $e) { if (!$e instanceof NotFound) { throw $e; } + $collectionSizeCache[$collectionId] = 0; } } + $value += $collectionSizeCache[$collectionId]; } + } - $diff = $value - $previousValue; - - // Project - $documents[] = new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => METRIC_DATABASES_STORAGE, - 'value' => $diff, - 'region' => System::getEnv('_APP_REGION', 'default'), - ]); + $diff = $value - $previousValue; + if ($diff === 0) { break; - } + } + + $keys = [ + METRIC_DATABASES_STORAGE + ]; + + foreach ($keys as $metric) { + $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); + } + + break; + default: + Console::log('Unknown storage level.'); + break; } + } - $dbForProject->createOrUpdateDocumentsWithIncrease( - collection: 'stats', - attribute: 'value', - documents: $documents - ); - - $end = microtime(true); - - Console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); + private function createStatsDocument( + string $id, + string $period, + string $time, + string $key, + int $diff, + ): Document + { + return new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $diff, + 'region' => System::getEnv('_APP_REGION', 'default'), + ]); } } From 77ae8dcc631e52c1643a1faa6c2001b0237897e2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 23:28:15 +1300 Subject: [PATCH 08/17] Lint --- src/Appwrite/Platform/Workers/UsageDump.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 3590398aaa..16f7f63321 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -117,8 +117,7 @@ class UsageDump extends Action array &$projectDocuments, array &$databaseCache, array &$collectionSizeCache, - ): void - { + ): void { $data = \explode('.', $key); $value = 0; $previousValue = 0; @@ -283,8 +282,7 @@ class UsageDump extends Action string $time, string $key, int $diff, - ): Document - { + ): Document { return new Document([ '$id' => $id, 'period' => $period, From fe930d8f646ad39ead1c6e87020ae3bf96a24017 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 00:07:45 +1300 Subject: [PATCH 09/17] Fix tests --- src/Appwrite/Platform/Workers/UsageDump.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 16f7f63321..7d11bab3ce 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -111,7 +111,7 @@ class UsageDump extends Action private function handleDatabaseStorage( string $id, string $key, - string $time, + ?string $time, string $period, Database $dbForProject, array &$projectDocuments, @@ -279,7 +279,7 @@ class UsageDump extends Action private function createStatsDocument( string $id, string $period, - string $time, + ?string $time, string $key, int $diff, ): Document { From 687be05d95454c67e8a23d48444ecf4528abfbf1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 00:08:04 +1300 Subject: [PATCH 10/17] Speed up dev mode usage tests --- .env | 2 +- tests/e2e/General/UsageTest.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 8f7d7996e7..c26d1ce7e0 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=30 +_APP_USAGE_AGGREGATION_INTERVAL=5 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index f6b2ca4882..4666922a47 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -23,7 +23,7 @@ class UsageTest extends Scope use SideServer; use FunctionsBase; - private const WAIT = 35; + private const WAIT = 5; private const CREATE = 20; protected string $projectId; From 91c319bc5070295360d397a8480e33d66b1bf0f5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 00:09:08 +1300 Subject: [PATCH 11/17] Remove redundant logs --- src/Appwrite/Platform/Workers/Usage.php | 16 ++++++---------- src/Appwrite/Platform/Workers/UsageDump.php | 8 +------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3687eeab67..3f7428d0dd 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -30,16 +30,13 @@ class Usage extends Action */ public function __construct() { - $this - ->desc('Usage worker') - ->inject('message') - ->inject('project') - ->inject('getProjectDB') - ->inject('queueForUsageDump') - ->callback(function (Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump) { - $this->action($message, $project, $getProjectDB, $queueForUsageDump); - }); + ->desc('Usage worker') + ->inject('message') + ->inject('project') + ->inject('getProjectDB') + ->inject('queueForUsageDump') + ->callback([$this, 'action']); $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $this->lastTriggeredTime = time(); @@ -61,7 +58,6 @@ class Usage extends Action throw new Exception('Missing payload'); } - if (empty($project->getAttribute('database'))) { var_dump($payload); return; diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 7d11bab3ce..b75d2ac47f 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -132,8 +132,6 @@ class UsageDump extends Action switch (\count($data)) { case METRIC_COLLECTION_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; $collectionInternalId = $data[1]; $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}"; @@ -168,7 +166,6 @@ class UsageDump extends Action break; case METRIC_DATABASE_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); $databaseInternalId = $data[0]; $databaseId = "database_{$databaseInternalId}"; @@ -268,10 +265,7 @@ class UsageDump extends Action foreach ($keys as $metric) { $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); } - - break; - default: - Console::log('Unknown storage level.'); + break; } } From 5dd3b09ca7d76464e6de4cfbf889dbb8d8b126b0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 00:13:13 +1300 Subject: [PATCH 12/17] Lint --- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index b75d2ac47f..3b9bc6cc31 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -265,7 +265,7 @@ class UsageDump extends Action foreach ($keys as $metric) { $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff); } - + break; } } From 03a8f350827136c3949be9ef8c34e0c7b4d13f79 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 00:14:30 +1300 Subject: [PATCH 13/17] Ensure wait overlaps aggregation interval --- tests/e2e/General/UsageTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 4666922a47..e03d853e90 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -23,7 +23,7 @@ class UsageTest extends Scope use SideServer; use FunctionsBase; - private const WAIT = 5; + private const WAIT = 7; private const CREATE = 20; protected string $projectId; From 8097f1607fb7a9532c266a1886beb2c70a1e52fc Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 01:34:44 +1300 Subject: [PATCH 14/17] Add project log --- src/Appwrite/Platform/Workers/UsageDump.php | 8 + tests/e2e/General/UsageTest.php | 205 +------------------- 2 files changed, 11 insertions(+), 202 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 3b9bc6cc31..74280ddfad 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -58,11 +58,19 @@ class UsageDump extends Action try { foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); + $numberOfKeys = !empty($stats['keys']) ? \count($stats['keys']) : 0; + $receivedAt = $stats['receivedAt'] ?? 'NONE'; + if ($numberOfKeys === 0) { + continue; + } + $dbForProject = $getProjectDB($project); $projectDocuments = []; $databaseCache = []; $collectionSizeCache = []; + Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { continue; diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index e03d853e90..a239625a96 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -651,200 +651,6 @@ class UsageTest extends Scope ]; } - // /** @depends testDatabaseStoragePrepare */ - // #[Retry(count: 1)] - // public function testDatabaseStorageStatsCreateDocument(array $data): array - // { - // $databaseId = $data['databaseId']; - // $collectionId = $data['collectionId']; - - // $originalProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $originalProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $originalProjectMetrics['body']); - - // $originalProjectMetrics = $originalProjectMetrics['body']; - - // $originalDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $originalDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $originalDatabaseMetrics['body']); - // $originalDatabaseMetrics = $originalDatabaseMetrics['body']; - - // // Create documents - // for ($i = 0; $i < 100; $i++) { - // $response = $this->client->call( - // Client::METHOD_POST, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', - // array_merge([ - // 'content-type' => 'application/json', - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'documentId' => 'unique()', - // 'data' => ['data' => str_repeat('a', 10000)], - // ] - // ); - - // $this->assertEquals(201, $response['headers']['status-code']); - // } - - // sleep(self::WAIT); - - // for ($i = 0; $i < 3; $i++) { - // try { - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); - // $this->assertGreaterThan($originalProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); - - // $newProjectMetrics = $newProjectMetrics['body']; - - // $newDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); - // $this->assertGreaterThan($originalDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); - - // $newDatabaseMetrics = $newDatabaseMetrics['body']; - - // return [ - // 'databaseId' => $databaseId, - // 'collectionId' => $collectionId, - // 'currentProjectMetrics' => $newProjectMetrics, - // 'currentDatabaseMetrics' => $newDatabaseMetrics, - // ]; - // } catch (ExpectationFailedException $e) { - // if ($i === 2) { - // throw $e; - // } - // sleep(self::WAIT); - // continue; - // } - // } - // } - - // /** @depends testDatabaseStorageStatsCreateDocument */ - // #[Retry(count: 1)] - // public function testDatabaseStorageStatsDeleteDocument(array $data): array - // { - // $databaseId = $data['databaseId']; - // $collectionId = $data['collectionId']; - // $currentProjectMetrics = $data['currentProjectMetrics']; - // $currentDatabaseMetrics = $data['currentDatabaseMetrics']; - - // $documents = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', - // array_merge([ - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()), - // [ - // 'queries' => [ - // Query::limit(50)->toString() - // ] - // ] - // ); - - // foreach ($documents['body']['documents'] as $document) { - // $response = $this->client->call( - // Client::METHOD_DELETE, - // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document['$id'], - // array_merge([ - // 'x-appwrite-project' => $this->getProject()['$id'] - // ], $this->getHeaders()) - // ); - - // $this->assertEquals(204, $response['headers']['status-code']); - // } - - // sleep(self::WAIT); - - // for ($i = 0; $i < 3; $i++) { - // try { - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - - // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); - // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); - // $this->assertLessThan($currentProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); - - // $newProjectMetrics = $newProjectMetrics['body']; - - // $newDatabaseMetrics = $this->client->call( - // Client::METHOD_GET, - // '/databases/' . $databaseId . '/usage?range=30d', - // $this->getConsoleHeaders() - // ); - - // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); - // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); - // $this->assertLessThan($currentDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); - - // $newDatabaseMetrics = $newDatabaseMetrics['body']; - - // return [ - // 'databaseId' => $databaseId, - // 'collectionId' => $collectionId, - // 'currentProjectMetrics' => $newProjectMetrics, - // 'currentDatabaseMetrics' => $newDatabaseMetrics, - // ]; - // } catch (ExpectationFailedException $e) { - // if ($i === 2) { - // throw $e; - // } - // sleep(self::WAIT); - // continue; - // } - // } - - // $newProjectMetrics = $this->client->call( - // Client::METHOD_GET, - // '/project/usage', - // $this->getConsoleHeaders(), - // [ - // 'period' => '1d', - // 'startDate' => self::getToday(), - // 'endDate' => self::getTomorrow(), - // ] - // ); - // } - /** @depends testDatabaseStats */ public function testPrepareFunctionsStats(array $data): array { @@ -905,7 +711,7 @@ class UsageTest extends Scope $this->assertEquals('index.php', $response['body']['entrypoint']); // Wait for deployment to build. - sleep(self::WAIT + 20); + sleep(self::WAIT); $response = $this->client->call( Client::METHOD_PATCH, @@ -976,18 +782,13 @@ class UsageTest extends Scope array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), - [ - 'async' => true, - ] + ], $this->getHeaders()) ); - $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals(200, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); $this->assertEquals($functionId, $response['body']['functionId']); - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/functions/' . $functionId . '/executions/' . $response['body']['$id'], From 1360c16f65cc015a7515761dfb68942fc5c578a6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 01:43:25 +1300 Subject: [PATCH 15/17] Partial revert "Add project log" This partially reverts commit 8097f1607fb7a9532c266a1886beb2c70a1e52fc. --- tests/e2e/General/UsageTest.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index a239625a96..ffc63708c8 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -711,7 +711,7 @@ class UsageTest extends Scope $this->assertEquals('index.php', $response['body']['entrypoint']); // Wait for deployment to build. - sleep(self::WAIT); + sleep(self::WAIT + 20); $response = $this->client->call( Client::METHOD_PATCH, @@ -782,13 +782,18 @@ class UsageTest extends Scope array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()) + ], $this->getHeaders()), + [ + 'async' => true, + ] ); - $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(202, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); $this->assertEquals($functionId, $response['body']['functionId']); + sleep(self::WAIT); + $response = $this->client->call( Client::METHOD_GET, '/functions/' . $functionId . '/executions/' . $response['body']['$id'], From f849fb9b30496af213d6aa04acb7425114e41f3e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 02:04:18 +1300 Subject: [PATCH 16/17] Add project job timing --- src/Appwrite/Platform/Workers/UsageDump.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 74280ddfad..bb1d605442 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -69,7 +69,8 @@ class UsageDump extends Action $databaseCache = []; $collectionSizeCache = []; - Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); + Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys . ' Started'); + $start = \microtime(true); foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { @@ -110,6 +111,9 @@ class UsageDump extends Action attribute: 'value', documents: $projectDocuments ); + + $end = \microtime(true); + Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s'); } } catch (\Exception $e) { Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage()); From 297098b252e1b9db3f7d58fae8bc94434cf53f3d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Jan 2025 03:23:25 +1300 Subject: [PATCH 17/17] Revert test timing changes --- .env | 2 +- tests/e2e/General/UsageTest.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index c26d1ce7e0..8f7d7996e7 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=5 +_APP_USAGE_AGGREGATION_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index ffc63708c8..af8bb19102 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -23,7 +23,7 @@ class UsageTest extends Scope use SideServer; use FunctionsBase; - private const WAIT = 7; + private const WAIT = 35; private const CREATE = 20; protected string $projectId;