diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5b7438de42..1d35fec3c7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,9 +8,33 @@ env: IMAGE: appwrite-dev CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }} -on: [pull_request] +on: [ pull_request ] jobs: + check_database_changes: + name: Check if utopia-php/database changed + runs-on: ubuntu-latest + outputs: + database_changed: ${{ steps.check.outputs.database_changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + + - name: Check for utopia-php/database changes + id: check + run: | + if git diff origin/${{ github.event.pull_request.base.ref }} HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then + echo "Database version changed, going to run all mode tests." + echo "database_changed=true" >> "$GITHUB_ENV" + echo "database_changed=true" >> "$GITHUB_OUTPUT" + else + echo "database_changed=false" >> "$GITHUB_ENV" + echo "database_changed=false" >> "$GITHUB_OUTPUT" + fi + setup: name: Setup & Build Appwrite Image runs-on: ubuntu-latest @@ -103,6 +127,62 @@ jobs: name: E2E Service Test runs-on: ubuntu-latest needs: setup + strategy: + fail-fast: false + matrix: + service: [ + Account, + Avatars, + Console, + Databases, + Functions, + FunctionsSchedule, + GraphQL, + Health, + Locale, + Projects, + Realtime, + Storage, + Teams, + Users, + Webhooks, + VCS, + Messaging, + Migrations + ] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Load Cache + uses: actions/cache@v4 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Load and Start Appwrite + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 30 + + - name: Run ${{ matrix.service }} tests with Project table mode + run: | + echo "Using project tables" + export _APP_DATABASE_SHARED_TABLES= + export _APP_DATABASE_SHARED_TABLES_V1= + + docker compose exec -T \ + -e _APP_DATABASE_SHARED_TABLES \ + -e _APP_DATABASE_SHARED_TABLES_V1 \ + appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug + + e2e_shared_mode_test: + name: E2E Shared Mode Service Test + runs-on: ubuntu-latest + needs: [ setup, check_database_changes ] + if: needs.check_database_changes.outputs.database_changed == 'true' strategy: fail-fast: false matrix: @@ -128,7 +208,6 @@ jobs: Migrations ] tables-mode: [ - 'Project', 'Shared V1', 'Shared V2', ] @@ -160,12 +239,8 @@ jobs: echo "Using shared tables V2" export _APP_DATABASE_SHARED_TABLES=database_db_main export _APP_DATABASE_SHARED_TABLES_V1= - else - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= fi - + docker compose exec -T \ -e _APP_DATABASE_SHARED_TABLES \ -e _APP_DATABASE_SHARED_TABLES_V1 \ @@ -251,4 +326,4 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body-path: benchmark.txt - edit-mode: replace \ No newline at end of file + edit-mode: replace diff --git a/composer.json b/composer.json index b7c3711410..58b74c3fe3 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.49.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.49.*", + "utopia-php/audit": "0.50.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index e653783ef5..fd89ba3ae8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4a54d0bd5973ed68082970f317664df3", + "content-hash": "884381b7cc6c225f83c397eb472ddf11", "packages": [ { "name": "adhocore/jwt", @@ -3474,16 +3474,16 @@ }, { "name": "utopia-php/audit", - "version": "0.49.0", + "version": "0.50.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d" + "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/c0da7dcdd35fc7d3f9640ba21cc82607cf7da729", + "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729", "shasum": "" }, "require": { @@ -3515,9 +3515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.49.0" + "source": "https://github.com/utopia-php/audit/tree/0.50.0" }, - "time": "2025-02-04T07:27:18+00:00" + "time": "2025-02-12T05:30:25+00:00" }, { "name": "utopia-php/cache", @@ -4607,16 +4607,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.8", + "version": "0.18.9", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "84737afa634e6a833fc4f8b0c967553234d3f215" + "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/84737afa634e6a833fc4f8b0c967553234d3f215", - "reference": "84737afa634e6a833fc4f8b0c967553234d3f215", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c", + "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c", "shasum": "" }, "require": { @@ -4656,9 +4656,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.8" + "source": "https://github.com/utopia-php/storage/tree/0.18.9" }, - "time": "2024-12-04T08:30:35+00:00" + "time": "2025-02-11T13:10:40+00:00" }, { "name": "utopia-php/swoole", @@ -8747,7 +8747,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index c0bcab1c3a..f1ae46eea7 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -6,15 +6,33 @@ use Appwrite\Auth\Auth; use Exception; use Throwable; use Utopia\Audit\Audit; +use Utopia\CLI\Console; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\System\System; class Audits extends Action { + private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + private const BATCH_SIZE_PRODUCTION = 5_000; + private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + + private int $lastTriggeredTime = 0; + + private array $logs = []; + + private function getBatchSize(): int + { + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; + } + public static function getName(): string { return 'audits'; @@ -30,6 +48,8 @@ class Audits extends Action ->inject('message') ->inject('dbForProject') ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject)); + + $this->lastTriggeredTime = time(); } @@ -44,13 +64,14 @@ class Audits extends Action */ public function action(Message $message, Database $dbForProject): void { - $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } + Console::info('Aggregating audit logs'); + $event = $payload['event'] ?? ''; $auditPayload = $payload['payload'] ?? ''; $mode = $payload['mode'] ?? ''; @@ -63,23 +84,48 @@ class Audits extends Action $userEmail = $user->getAttribute('email', ''); $userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER); - $audit = new Audit($dbForProject); - $audit->log( - userId: $user->getInternalId(), - // Pass first, most verbose event pattern - event: $event, - resource: $resource, - userAgent: $userAgent, - ip: $ip, - location: '', - data: [ + // Create event data + $eventData = [ + 'userId' => $user->getInternalId(), + 'event' => $event, + 'resource' => $resource, + 'userAgent' => $userAgent, + 'ip' => $ip, + 'location' => '', + 'data' => [ 'userId' => $user->getId(), 'userName' => $userName, 'userEmail' => $userEmail, 'userType' => $userType, 'mode' => $mode, 'data' => $auditPayload, - ] - ); + ], + 'timestamp' => DateTime::formatTz(DateTime::now()) + ]; + + $this->logs[] = $eventData; + + // Check if we should process the batch by checking both for the batch size and the elapsed time + $batchSize = $this->getBatchSize(); + $shouldProcessBatch = count($this->logs) >= $batchSize; + if (!$shouldProcessBatch && count($this->logs) > 0) { + $shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; + } + + if ($shouldProcessBatch) { + Console::log('Processing batch with ' . count($this->logs) . ' events'); + $audit = new Audit($dbForProject); + + try { + $audit->logBatch($this->logs); + Console::success('Audit logs processed successfully'); + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + // Clear the pending events after successful batch processing + $this->logs = []; + $this->lastTriggeredTime = time(); + } + } } } diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index e7cbbd5088..c21c28b517 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -54,8 +54,10 @@ class Builds extends Action ->inject('cache') ->inject('dbForProject') ->inject('deviceForFunctions') + ->inject('isResourceBlocked') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => + $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); } /** @@ -72,7 +74,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void { $payload = $message->getPayload() ?? []; @@ -93,7 +95,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); + $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); break; default: @@ -118,7 +120,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -127,7 +129,11 @@ class Builds extends Action $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { - throw new \Exception('Function not found', 404); + throw new \Exception('Function not found'); + } + + if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $functionId)) { + throw new \Exception('Function blocked'); } $deploymentId = $deployment->getId(); @@ -135,11 +141,11 @@ class Builds extends Action $deployment = $dbForProject->getDocument('deployments', $deploymentId); if ($deployment->isEmpty()) { - throw new \Exception('Deployment not found', 404); + throw new \Exception('Deployment not found'); } if (empty($deployment->getAttribute('entrypoint', ''))) { - throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500); + throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".'); } $version = $function->getAttribute('version', 'v2'); @@ -571,7 +577,7 @@ class Builds extends Action $build = $dbForProject->getDocument('builds', $build->getId()); if ($build->isEmpty()) { - throw new \Exception('Build not found', 404); + throw new \Exception('Build not found'); } if ($build->getAttribute('status') === 'canceled') { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 46ae480684..08c6fc1b72 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -7,6 +7,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; +use Utopia\Abuse\Adapters\TimeLimit\Database as AbuseDatabase; use Utopia\Audit\Audit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -505,7 +506,8 @@ class Deletes extends Action $projectCollectionIds = [ ...\array_keys(Config::getParam('collections', [])['projects']), - Audit::COLLECTION + Audit::COLLECTION, + AbuseDatabase::COLLECTION, ]; $limit = \count($projectCollectionIds) + 25; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index e3c76ecb9a..639e90a867 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -160,6 +160,12 @@ class StatsResources extends Action call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); } + try { + $this->countImageTransformations($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); + } + try { $this->countForDatabase($dbForProject, $dbForLogs, $region); } catch (Throwable $th) { @@ -246,6 +252,8 @@ class StatsResources extends Action }); $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations, 'inf'); + $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalDailyImageTransformations, '1d'); + $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalHourlyImageTransformations, '1h'); }