Merge branch '1.6.x' into feat-remove-deprecated-usage-stuff

2026-05-22 16:38:32 +00:00 · 2025-02-13 06:28:48 +05:45 · 2025-02-13 06:28:48 +05:45 · 29ffd703db
commit 29ffd703db
parent f2fc45da02 88c2fe2dd2
7 changed files with 182 additions and 45 deletions
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@ -8,9 +8,33 @@ env:
  IMAGE: appwrite-dev
  CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }}

-on: [pull_request]
+on: [ pull_request ]

 jobs:
+  check_database_changes:
+    name: Check if utopia-php/database changed
+    runs-on: ubuntu-latest
+    outputs:
+      database_changed: ${{ steps.check.outputs.database_changed }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Fetch base branch
+        run: git fetch origin ${{ github.event.pull_request.base.ref }}
+
+      - name: Check for utopia-php/database changes
+        id: check
+        run: |
+          if git diff origin/${{ github.event.pull_request.base.ref }} HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then
+            echo "Database version changed, going to run all mode tests."
+            echo "database_changed=true" >> "$GITHUB_ENV"
+            echo "database_changed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "database_changed=false" >> "$GITHUB_ENV"
+            echo "database_changed=false" >> "$GITHUB_OUTPUT"
+          fi
+
  setup:
    name: Setup & Build Appwrite Image
    runs-on: ubuntu-latest
@ -103,6 +127,62 @@ jobs:
    name: E2E Service Test
    runs-on: ubuntu-latest
    needs: setup
+    strategy:
+      fail-fast: false
+      matrix:
+        service: [
+          Account,
+          Avatars,
+          Console,
+          Databases,
+          Functions,
+          FunctionsSchedule,
+          GraphQL,
+          Health,
+          Locale,
+          Projects,
+          Realtime,
+          Storage,
+          Teams,
+          Users,
+          Webhooks,
+          VCS,
+          Messaging,
+          Migrations
+        ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Load Cache
+        uses: actions/cache@v4
+        with:
+          key: ${{ env.CACHE_KEY }}
+          path: /tmp/${{ env.IMAGE }}.tar
+          fail-on-cache-miss: true
+
+      - name: Load and Start Appwrite
+        run: |
+          docker load --input /tmp/${{ env.IMAGE }}.tar
+          docker compose up -d
+          sleep 30
+
+      - name: Run ${{ matrix.service }} tests with Project table mode
+        run: |
+          echo "Using project tables"
+          export _APP_DATABASE_SHARED_TABLES=
+          export _APP_DATABASE_SHARED_TABLES_V1=
+
+          docker compose exec -T \
+            -e _APP_DATABASE_SHARED_TABLES \
+            -e _APP_DATABASE_SHARED_TABLES_V1 \
+            appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug
+
+  e2e_shared_mode_test:
+    name: E2E Shared Mode Service Test
+    runs-on: ubuntu-latest
+    needs: [ setup, check_database_changes ]
+    if: needs.check_database_changes.outputs.database_changed == 'true'
    strategy:
      fail-fast: false
      matrix:
@ -128,7 +208,6 @@ jobs:
            Migrations
          ]
        tables-mode: [
-          'Project',
          'Shared V1',
          'Shared V2',
        ]
@ -160,12 +239,8 @@ jobs:
            echo "Using shared tables V2"
            export _APP_DATABASE_SHARED_TABLES=database_db_main
            export _APP_DATABASE_SHARED_TABLES_V1=
-          else
-            echo "Using project tables"
-            export _APP_DATABASE_SHARED_TABLES=
-            export _APP_DATABASE_SHARED_TABLES_V1=
          fi
-      
+          
          docker compose exec -T \
            -e _APP_DATABASE_SHARED_TABLES \
            -e _APP_DATABASE_SHARED_TABLES_V1 \
@ -251,4 +326,4 @@ jobs:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body-path: benchmark.txt
-          edit-mode: replace
+          edit-mode: replace
--- a/composer.json
+++ b/composer.json
@ -47,7 +47,7 @@
        "appwrite/php-clamav": "2.0.*",
        "utopia-php/abuse": "0.49.*",
        "utopia-php/analytics": "0.10.*",
-        "utopia-php/audit": "0.49.*",
+        "utopia-php/audit": "0.50.*",
        "utopia-php/cache": "0.11.*",
        "utopia-php/cli": "0.15.*",
        "utopia-php/config": "0.2.*",
--- a/composer.lock
+++ b/composer.lock
@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "4a54d0bd5973ed68082970f317664df3",
+    "content-hash": "884381b7cc6c225f83c397eb472ddf11",
    "packages": [
        {
            "name": "adhocore/jwt",
@ -3474,16 +3474,16 @@
        },
        {
            "name": "utopia-php/audit",
-            "version": "0.49.0",
+            "version": "0.50.0",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/audit.git",
-                "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d"
+                "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/audit/zipball/9d5c5e0cf0f6d9157b911fc3971da4331d71c96d",
-                "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d",
+                "url": "https://api.github.com/repos/utopia-php/audit/zipball/c0da7dcdd35fc7d3f9640ba21cc82607cf7da729",
+                "reference": "c0da7dcdd35fc7d3f9640ba21cc82607cf7da729",
                "shasum": ""
            },
            "require": {
@ -3515,9 +3515,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/audit/issues",
-                "source": "https://github.com/utopia-php/audit/tree/0.49.0"
+                "source": "https://github.com/utopia-php/audit/tree/0.50.0"
            },
-            "time": "2025-02-04T07:27:18+00:00"
+            "time": "2025-02-12T05:30:25+00:00"
        },
        {
            "name": "utopia-php/cache",
@ -4607,16 +4607,16 @@
        },
        {
            "name": "utopia-php/storage",
-            "version": "0.18.8",
+            "version": "0.18.9",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/storage.git",
-                "reference": "84737afa634e6a833fc4f8b0c967553234d3f215"
+                "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/storage/zipball/84737afa634e6a833fc4f8b0c967553234d3f215",
-                "reference": "84737afa634e6a833fc4f8b0c967553234d3f215",
+                "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c",
+                "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c",
                "shasum": ""
            },
            "require": {
@ -4656,9 +4656,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/storage/issues",
-                "source": "https://github.com/utopia-php/storage/tree/0.18.8"
+                "source": "https://github.com/utopia-php/storage/tree/0.18.9"
            },
-            "time": "2024-12-04T08:30:35+00:00"
+            "time": "2025-02-11T13:10:40+00:00"
        },
        {
            "name": "utopia-php/swoole",
@ -8747,7 +8747,7 @@
    ],
    "aliases": [],
    "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {},
    "prefer-stable": false,
    "prefer-lowest": false,
    "platform": {
--- a/src/Appwrite/Platform/Workers/Audits.php
+++ b/src/Appwrite/Platform/Workers/Audits.php
@ -6,15 +6,33 @@ use Appwrite\Auth\Auth;
 use Exception;
 use Throwable;
 use Utopia\Audit\Audit;
+use Utopia\CLI\Console;
 use Utopia\Database\Database;
+use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception\Authorization;
 use Utopia\Database\Exception\Structure;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
+use Utopia\System\System;

 class Audits extends Action
 {
+    private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development
+    private const BATCH_SIZE_PRODUCTION = 5_000;
+    private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds
+
+    private int $lastTriggeredTime = 0;
+
+    private array $logs = [];
+
+    private function getBatchSize(): int
+    {
+        return System::getEnv('_APP_ENV', 'development') === 'development'
+            ? self::BATCH_SIZE_DEVELOPMENT
+            : self::BATCH_SIZE_PRODUCTION;
+    }
+
    public static function getName(): string
    {
        return 'audits';
@ -30,6 +48,8 @@ class Audits extends Action
            ->inject('message')
            ->inject('dbForProject')
            ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject));
+
+        $this->lastTriggeredTime = time();
    }


@ -44,13 +64,14 @@ class Audits extends Action
     */
    public function action(Message $message, Database $dbForProject): void
    {
-
        $payload = $message->getPayload() ?? [];

        if (empty($payload)) {
            throw new Exception('Missing payload');
        }

+        Console::info('Aggregating audit logs');
+
        $event = $payload['event'] ?? '';
        $auditPayload = $payload['payload'] ?? '';
        $mode = $payload['mode'] ?? '';
@ -63,23 +84,48 @@ class Audits extends Action
        $userEmail = $user->getAttribute('email', '');
        $userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER);

-        $audit = new Audit($dbForProject);
-        $audit->log(
-            userId: $user->getInternalId(),
-            // Pass first, most verbose event pattern
-            event: $event,
-            resource: $resource,
-            userAgent: $userAgent,
-            ip: $ip,
-            location: '',
-            data: [
+        // Create event data
+        $eventData = [
+            'userId' => $user->getInternalId(),
+            'event' => $event,
+            'resource' => $resource,
+            'userAgent' => $userAgent,
+            'ip' => $ip,
+            'location' => '',
+            'data' => [
                'userId' => $user->getId(),
                'userName' => $userName,
                'userEmail' => $userEmail,
                'userType' => $userType,
                'mode' => $mode,
                'data' => $auditPayload,
-            ]
-        );
+            ],
+            'timestamp' => DateTime::formatTz(DateTime::now())
+        ];
+
+        $this->logs[] = $eventData;
+
+        // Check if we should process the batch by checking both for the batch size and the elapsed time
+        $batchSize = $this->getBatchSize();
+        $shouldProcessBatch = count($this->logs) >= $batchSize;
+        if (!$shouldProcessBatch && count($this->logs) > 0) {
+            $shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
+        }
+
+        if ($shouldProcessBatch) {
+            Console::log('Processing batch with ' . count($this->logs) . ' events');
+            $audit = new Audit($dbForProject);
+
+            try {
+                $audit->logBatch($this->logs);
+                Console::success('Audit logs processed successfully');
+            } catch (Throwable $e) {
+                Console::error('Error processing audit logs: ' . $e->getMessage());
+            } finally {
+                // Clear the pending events after successful batch processing
+                $this->logs = [];
+                $this->lastTriggeredTime = time();
+            }
+        }
    }
 }
--- a/src/Appwrite/Platform/Workers/Builds.php
+++ b/src/Appwrite/Platform/Workers/Builds.php
@ -54,8 +54,10 @@ class Builds extends Action
            ->inject('cache')
            ->inject('dbForProject')
            ->inject('deviceForFunctions')
+            ->inject('isResourceBlocked')
            ->inject('log')
-            ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
+            ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) =>
+                $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log));
    }

    /**
@ -72,7 +74,7 @@ class Builds extends Action
     * @return void
     * @throws \Utopia\Database\Exception
     */
-    public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
+    public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void
    {
        $payload = $message->getPayload() ?? [];

@ -93,7 +95,7 @@ class Builds extends Action
            case BUILD_TYPE_RETRY:
                Console::info('Creating build for deployment: ' . $deployment->getId());
                $github = new GitHub($cache);
-                $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
+                $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
                break;

            default:
@ -118,7 +120,7 @@ class Builds extends Action
     * @throws \Utopia\Database\Exception
     * @throws Exception
     */
-    protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
+    protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
    {
        $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));

@ -127,7 +129,11 @@ class Builds extends Action

        $function = $dbForProject->getDocument('functions', $functionId);
        if ($function->isEmpty()) {
-            throw new \Exception('Function not found', 404);
+            throw new \Exception('Function not found');
+        }
+
+        if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $functionId)) {
+            throw new \Exception('Function blocked');
        }

        $deploymentId = $deployment->getId();
@ -135,11 +141,11 @@ class Builds extends Action

        $deployment = $dbForProject->getDocument('deployments', $deploymentId);
        if ($deployment->isEmpty()) {
-            throw new \Exception('Deployment not found', 404);
+            throw new \Exception('Deployment not found');
        }

        if (empty($deployment->getAttribute('entrypoint', ''))) {
-            throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
+            throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".');
        }

        $version = $function->getAttribute('version', 'v2');
@ -571,7 +577,7 @@ class Builds extends Action
                                    $build = $dbForProject->getDocument('builds', $build->getId());

                                    if ($build->isEmpty()) {
-                                        throw new \Exception('Build not found', 404);
+                                        throw new \Exception('Build not found');
                                    }

                                    if ($build->getAttribute('status') === 'canceled') {
--- a/src/Appwrite/Platform/Workers/Deletes.php
+++ b/src/Appwrite/Platform/Workers/Deletes.php
@ -7,6 +7,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter;
 use Appwrite\Extend\Exception;
 use Executor\Executor;
 use Throwable;
+use Utopia\Abuse\Adapters\TimeLimit\Database as AbuseDatabase;
 use Utopia\Audit\Audit;
 use Utopia\Cache\Adapter\Filesystem;
 use Utopia\Cache\Cache;
@ -505,7 +506,8 @@ class Deletes extends Action

        $projectCollectionIds = [
            ...\array_keys(Config::getParam('collections', [])['projects']),
-            Audit::COLLECTION
+            Audit::COLLECTION,
+            AbuseDatabase::COLLECTION,
        ];

        $limit = \count($projectCollectionIds) + 25;
--- a/src/Appwrite/Platform/Workers/StatsResources.php
+++ b/src/Appwrite/Platform/Workers/StatsResources.php
@ -160,6 +160,12 @@ class StatsResources extends Action
                call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
            }

+            try {
+                $this->countImageTransformations($dbForProject, $dbForLogs, $region);
+            } catch (Throwable $th) {
+                call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
+            }
+
            try {
                $this->countForDatabase($dbForProject, $dbForLogs, $region);
            } catch (Throwable $th) {
@ -246,6 +252,8 @@ class StatsResources extends Action
        });

        $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations, 'inf');
+        $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalDailyImageTransformations, '1d');
+        $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalHourlyImageTransformations, '1h');

    }