Merge branch '1.6.x' of https://github.com/appwrite/appwrite into fix-alter-attributes

# Conflicts:
#	src/Appwrite/Platform/Workers/Databases.php

.gitignore

@@ -16,3 +16,4 @@ dev/yasd_init.php
 .phpunit.result.cache
 Makefile
 appwrite.json
+.zed/

app/controllers/api/teams.php

@@ -805,8 +805,11 @@ App::get('/v1/teams/:teamId/memberships')
         }, $membershipsPrivacy);
 
         $memberships = array_map(function ($membership) use ($dbForProject, $team, $membershipsPrivacy) {
+            $user = !empty(array_filter($membershipsPrivacy))
+                ? $dbForProject->getDocument('users', $membership->getAttribute('userId'))
+                : new Document();
+
             if ($membershipsPrivacy['mfa']) {
-                $user = $dbForProject->getDocument('users', $membership->getAttribute('userId'));
                 $mfa = $user->getAttribute('mfa', false);
 
                 if ($mfa) {
@@ -888,9 +891,11 @@ App::get('/v1/teams/:teamId/memberships/:membershipId')
             return $privacy || $isPrivilegedUser || $isAppUser;
         }, $membershipsPrivacy);
 
-        if ($membershipsPrivacy['mfa']) {
-            $user = $dbForProject->getDocument('users', $membership->getAttribute('userId'));
+        $user = !empty(array_filter($membershipsPrivacy))
+            ? $dbForProject->getDocument('users', $membership->getAttribute('userId'))
+            : new Document();
 
+        if ($membershipsPrivacy['mfa']) {
             $mfa = $user->getAttribute('mfa', false);
 
             if ($mfa) {

app/realtime.php

@@ -29,6 +29,7 @@ use Utopia\Database\Validator\Authorization;
 use Utopia\DSN\DSN;
 use Utopia\Logger\Log;
 use Utopia\System\System;
+use Utopia\Telemetry\Adapter\None as NoTelemetry;
 use Utopia\WebSocket\Adapter;
 use Utopia\WebSocket\Server;
 
@@ -142,6 +143,13 @@ if (!function_exists('getRealtime')) {
     }
 }
 
+if (!function_exists('getTelemetry')) {
+    function getTelemetry(int $workerId): Utopia\Telemetry\Adapter
+    {
+        return new NoTelemetry();
+    }
+}
+
 $realtime = getRealtime();
 
 /**
@@ -274,6 +282,12 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
 $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $logError) {
     Console::success('Worker ' . $workerId . ' started successfully');
 
+    $telemetry = getTelemetry($workerId);
+    $register->set('telemetry', fn () => $telemetry);
+    $register->set('telemetry.connectionCounter', fn () => $telemetry->createUpDownCounter('realtime.server.open_connections'));
+    $register->set('telemetry.connectionCreatedCounter', fn () => $telemetry->createCounter('realtime.server.connection.created'));
+    $register->set('telemetry.messageSentCounter', fn () => $telemetry->createCounter('realtime.server.message.sent'));
+
     $attempts = 0;
     $start = time();
 
@@ -416,6 +430,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
                 );
 
                 if (($num = count($receivers)) > 0) {
+                    $register->get('telemetry.messageSentCounter')->add($num);
                     $stats->incr($event['project'], 'messages', $num);
                 }
             });
@@ -519,6 +534,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
             ]
         ]));
 
+        $register->get('telemetry.connectionCounter')->add(1);
+        $register->get('telemetry.connectionCreatedCounter')->add(1);
+
         $stats->set($project->getId(), [
             'projectId' => $project->getId(),
             'teamId' => $project->getAttribute('teamId')
@@ -592,9 +610,12 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
         }
 
         switch ($message['type']) {
-            /**
-             * This type is used to authenticate.
-             */
+            case 'ping':
+                $server->send([$connection], json_encode([
+                    'type' => 'pong'
+                ]));
+
+                break;
             case 'authentication':
                 if (!array_key_exists('session', $message['data'])) {
                     throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.');
@@ -652,12 +673,14 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
     }
 });
 
-$server->onClose(function (int $connection) use ($realtime, $stats) {
+$server->onClose(function (int $connection) use ($realtime, $stats, $register) {
     if (array_key_exists($connection, $realtime->connections)) {
         $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal');
     }
     $realtime->unsubscribe($connection);
 
+    $register->get('telemetry.connectionCounter')->add(-1);
+
     Console::info('Connection close: ' . $connection);
 });
 

composer.json

@@ -70,6 +70,7 @@
         "utopia-php/storage": "0.18.*",
         "utopia-php/swoole": "0.8.*",
         "utopia-php/system": "0.9.*",
+        "utopia-php/telemetry": "0.1.*",
         "utopia-php/vcs": "0.8.*",
         "utopia-php/websocket": "0.1.*",
         "matomo/device-detector": "6.1.*",
@@ -96,6 +97,10 @@
     "config": {
         "platform": {
             "php": "8.3"
+        },
+        "allow-plugins": {
+            "php-http/discovery": false,
+            "tbachert/spi": false
         }
     }
 }

docs/references/account/create-token-magic-url.md

@@ -1,3 +1,3 @@
-Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST /v1/account/sessions/token](https://appwrite.io/docs/references/cloud/client-web/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.
+Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST /v1/account/sessions/token](https://appwrite.io/docs/references/cloud/client-web/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.
 
 A user is limited to 10 active sessions at a time by default. [Learn more about session limits](https://appwrite.io/docs/authentication-security#limits).

src/Appwrite/Platform/Workers/Databases.php

@@ -92,6 +92,7 @@ class Databases extends Action
      * @throws Authorization
      * @throws Conflict
      * @throws \Exception
+     * @throws \Throwable
      */
     private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
@@ -134,7 +135,6 @@ class Databases extends Action
         $options = $attribute->getAttribute('options', []);
         $project = $dbForConsole->getDocument('projects', $projectId);
 
-
         try {
             switch ($type) {
                 case Database::VAR_RELATIONSHIP:
@@ -170,7 +170,6 @@ class Databases extends Action
 
             $dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available'));
         } catch (\Throwable $e) {
-            // TODO: Send non DatabaseExceptions to Sentry
             Console::error($e->getMessage());
 
             if ($e instanceof DatabaseException) {
@@ -193,15 +192,17 @@ class Databases extends Action
                     $relatedAttribute->setAttribute('status', 'failed')
                 );
             }
+
+            throw $e;
         } finally {
             $this->trigger($database, $collection, $attribute, $project, $projectId, $events);
-        }
 
-        if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) {
-            $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
-        }
+            if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) {
+                $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
+            }
 
-        $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
+            $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
+        }
     }
 
     /**
@@ -215,6 +216,7 @@ class Databases extends Action
      * @throws Authorization
      * @throws Conflict
      * @throws \Exception
+     * @throws \Throwable
      **/
     private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
@@ -248,111 +250,114 @@ class Databases extends Action
         // - stuck: attribute was available but cannot be removed
 
         try {
-            if ($status !== 'failed') {
-                if ($type === Database::VAR_RELATIONSHIP) {
-                    if ($options['twoWay']) {
-                        $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
-                        if ($relatedCollection->isEmpty()) {
-                            throw new DatabaseException('Collection not found');
+            try {
+                if ($status !== 'failed') {
+                    if ($type === Database::VAR_RELATIONSHIP) {
+                        if ($options['twoWay']) {
+                            $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
+                            if ($relatedCollection->isEmpty()) {
+                                throw new DatabaseException('Collection not found');
+                            }
+                            $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
                         }
-                        $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
-                    }
 
-                    if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
-                        $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
-                        throw new DatabaseException('Failed to delete Relationship');
+                        if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
+                            $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
+                            throw new DatabaseException('Failed to delete Relationship');
+                        }
+                    } elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
+                        throw new DatabaseException('Failed to delete Attribute');
                     }
-                } elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
-                    throw new DatabaseException('Failed to delete Attribute');
                 }
-            }
 
-            $dbForProject->deleteDocument('attributes', $attribute->getId());
+                $dbForProject->deleteDocument('attributes', $attribute->getId());
 
-            if (!$relatedAttribute->isEmpty()) {
-                $dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
-            }
-        } catch (\Throwable $e) {
-            // TODO: Send non DatabaseExceptions to Sentry
-            Console::error($e->getMessage());
-
-            if ($e instanceof DatabaseException) {
-                $attribute->setAttribute('error', $e->getMessage());
                 if (!$relatedAttribute->isEmpty()) {
-                    $relatedAttribute->setAttribute('error', $e->getMessage());
+                    $dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
+                }
+            } catch (\Throwable $e) {
+                Console::error($e->getMessage());
+
+                if ($e instanceof DatabaseException) {
+                    $attribute->setAttribute('error', $e->getMessage());
+                    if (!$relatedAttribute->isEmpty()) {
+                        $relatedAttribute->setAttribute('error', $e->getMessage());
+                    }
                 }
-            }
-            $dbForProject->updateDocument(
-                'attributes',
-                $attribute->getId(),
-                $attribute->setAttribute('status', 'stuck')
-            );
-            if (!$relatedAttribute->isEmpty()) {
                 $dbForProject->updateDocument(
                     'attributes',
-                    $relatedAttribute->getId(),
-                    $relatedAttribute->setAttribute('status', 'stuck')
+                    $attribute->getId(),
+                    $attribute->setAttribute('status', 'stuck')
                 );
+                if (!$relatedAttribute->isEmpty()) {
+                    $dbForProject->updateDocument(
+                        'attributes',
+                        $relatedAttribute->getId(),
+                        $relatedAttribute->setAttribute('status', 'stuck')
+                    );
+                }
+
+                throw $e;
+            } finally {
+                $this->trigger($database, $collection, $attribute, $project, $projectId, $events);
             }
-        } finally {
-            $this->trigger($database, $collection, $attribute, $project, $projectId, $events);
-        }
 
-        // The underlying database removes/rebuilds indexes when attribute is removed
-        // Update indexes table with changes
-        /** @var Document[] $indexes */
-        $indexes = $collection->getAttribute('indexes', []);
+            // The underlying database removes/rebuilds indexes when attribute is removed
+            // Update indexes table with changes
+            /** @var Document[] $indexes */
+            $indexes = $collection->getAttribute('indexes', []);
 
-        foreach ($indexes as $index) {
-            /** @var string[] $attributes */
-            $attributes = $index->getAttribute('attributes');
-            $lengths = $index->getAttribute('lengths');
-            $orders = $index->getAttribute('orders');
+            foreach ($indexes as $index) {
+                /** @var string[] $attributes */
+                $attributes = $index->getAttribute('attributes');
+                $lengths = $index->getAttribute('lengths');
+                $orders = $index->getAttribute('orders');
 
-            $found = \array_search($key, $attributes);
+                $found = \array_search($key, $attributes);
 
-            if ($found !== false) {
-                // If found, remove entry from attributes, lengths, and orders
-                // array_values wraps array_diff to reindex array keys
-                // when found attribute is removed from array
-                $attributes = \array_values(\array_diff($attributes, [$attributes[$found]]));
-                $lengths = \array_values(\array_diff($lengths, isset($lengths[$found]) ? [$lengths[$found]] : []));
-                $orders = \array_values(\array_diff($orders, isset($orders[$found]) ? [$orders[$found]] : []));
+                if ($found !== false) {
+                    // If found, remove entry from attributes, lengths, and orders
+                    // array_values wraps array_diff to reindex array keys
+                    // when found attribute is removed from array
+                    $attributes = \array_values(\array_diff($attributes, [$attributes[$found]]));
+                    $lengths = \array_values(\array_diff($lengths, isset($lengths[$found]) ? [$lengths[$found]] : []));
+                    $orders = \array_values(\array_diff($orders, isset($orders[$found]) ? [$orders[$found]] : []));
 
-                if (empty($attributes)) {
-                    $dbForProject->deleteDocument('indexes', $index->getId());
-                } else {
-                    $index
-                        ->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN)
-                        ->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN)
-                        ->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN);
-
-                    // Check if an index exists with the same attributes and orders
-                    $exists = false;
-                    foreach ($indexes as $existing) {
-                        if (
-                            $existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself
-                            && $existing->getAttribute('attributes') === $index->getAttribute('attributes')
-                            && $existing->getAttribute('orders') === $index->getAttribute('orders')
-                        ) {
-                            $exists = true;
-                            break;
-                        }
-                    }
-
-                    if ($exists) { // Delete the duplicate if created, else update in db
-                        $this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject);
+                    if (empty($attributes)) {
+                        $dbForProject->deleteDocument('indexes', $index->getId());
                     } else {
-                        $dbForProject->updateDocument('indexes', $index->getId(), $index);
+                        $index
+                            ->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN)
+                            ->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN)
+                            ->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN);
+
+                        // Check if an index exists with the same attributes and orders
+                        $exists = false;
+                        foreach ($indexes as $existing) {
+                            if (
+                                $existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself
+                                && $existing->getAttribute('attributes') === $index->getAttribute('attributes')
+                                && $existing->getAttribute('orders') === $index->getAttribute('orders')
+                            ) {
+                                $exists = true;
+                                break;
+                            }
+                        }
+
+                        if ($exists) { // Delete the duplicate if created, else update in db
+                            $this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject);
+                        } else {
+                            $dbForProject->updateDocument('indexes', $index->getId(), $index);
+                        }
                     }
                 }
             }
-        }
+        } finally {
+            $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
 
-        $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
-
-        if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) {
-            $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
+            if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) {
+                $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
+            }
         }
     }
 
@@ -368,6 +373,7 @@ class Databases extends Action
      * @throws Conflict
      * @throws Structure
      * @throws DatabaseException
+     * @throws \Throwable
      */
     private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
@@ -399,9 +405,7 @@ class Databases extends Action
             }
             $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available'));
         } catch (\Throwable $e) {
-            // TODO: Send non DatabaseExceptions to Sentry
             Console::error($e->getMessage());
-
             if ($e instanceof DatabaseException) {
                 $index->setAttribute('error', $e->getMessage());
             }
@@ -410,11 +414,12 @@ class Databases extends Action
                 $index->getId(),
                 $index->setAttribute('status', 'failed')
             );
+
+            throw $e;
         } finally {
             $this->trigger($database, $collection, $index, $project, $projectId, $events);
+            $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
         }
-
-        $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
     }
 
     /**
@@ -429,6 +434,7 @@ class Databases extends Action
      * @throws Conflict
      * @throws Structure
      * @throws DatabaseException
+     * @throws \Throwable
      */
     private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
@@ -457,7 +463,6 @@ class Databases extends Action
             $dbForProject->deleteDocument('indexes', $index->getId());
             $index->setAttribute('status', 'deleted');
         } catch (\Throwable $e) {
-            // TODO: Send non DatabaseExceptions to Sentry
             Console::error($e->getMessage());
 
             if ($e instanceof DatabaseException) {
@@ -468,11 +473,13 @@ class Databases extends Action
                 $index->getId(),
                 $index->setAttribute('status', 'stuck')
             );
+
+            throw $e;
+
         } finally {
             $this->trigger($database, $collection, $index, $project, $projectId, $events);
+            $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId());
         }
-
-        $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId());
     }
 
     /**

src/Appwrite/Platform/Workers/Migrations.php

@@ -268,8 +268,6 @@ class Migrations extends Action
         $transfer = $source = $destination = null;
 
         try {
-            $migration = $this->dbForProject->getDocument('migrations', $migration->getId());
-
             if (
                 $migration->getAttribute('source') === SourceAppwrite::getName() &&
                 empty($migration->getAttribute('credentials', []))

tests/e2e/Services/Realtime/RealtimeCustomClientTest.php

@@ -111,6 +111,30 @@ class RealtimeCustomClientTest extends Scope
         $client->close();
     }
 
+    public function testPingPong()
+    {
+        $client = $this->getWebsocket(['files'], [
+            'origin' => 'http://localhost'
+        ]);
+        $response = json_decode($client->receive(), true);
+
+        $this->assertArrayHasKey('type', $response);
+        $this->assertArrayHasKey('data', $response);
+        $this->assertEquals('connected', $response['type']);
+        $this->assertNotEmpty($response['data']);
+        $this->assertCount(1, $response['data']['channels']);
+        $this->assertContains('files', $response['data']['channels']);
+
+        $client->send(\json_encode([
+            'type' => 'ping'
+        ]));
+
+        $response = json_decode($client->receive(), true);
+        $this->assertEquals('pong', $response['type']);
+
+        $client->close();
+    }
+
     public function testManualAuthentication()
     {
         $user = $this->getUser();

tests/e2e/Services/Teams/TeamsCustomClientTest.php

@@ -83,6 +83,38 @@ class TeamsCustomClientTest extends Scope
         $this->assertNotEmpty($response['body']['memberships'][0]['userName']);
         $this->assertNotEmpty($response['body']['memberships'][0]['userEmail']);
         $this->assertArrayHasKey('mfa', $response['body']['memberships'][0]);
+
+        /**
+         * Update project settings to show only MFA
+         */
+        $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/memberships-privacy', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => 'console',
+            'cookie' => 'a_session_console=' . $this->getRoot()['session'],
+        ]), [
+            'userName' => false,
+            'userEmail' => false,
+            'mfa' => true,
+        ]);
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+
+        /**
+         * Test that sensitive fields are not shown
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $projectId,
+        ], $this->getHeaders()));
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+        $this->assertIsInt($response['body']['total']);
+        $this->assertNotEmpty($response['body']['memberships'][0]['$id']);
+
+        // Assert that sensitive fields are present
+        $this->assertEmpty($response['body']['memberships'][0]['userName']);
+        $this->assertEmpty($response['body']['memberships'][0]['userEmail']);
+        $this->assertArrayHasKey('mfa', $response['body']['memberships'][0]);
     }
 
     /**