diff --git a/.github/workflows/sdk-preview.yml b/.github/workflows/sdk-preview.yml new file mode 100644 index 0000000000..92b4f454cb --- /dev/null +++ b/.github/workflows/sdk-preview.yml @@ -0,0 +1,33 @@ +name: "Console SDK Preview" + +on: + pull_request: + paths: + - 'app/config/specs/*-latest-console.json' + + +jobs: + setup: + name: Setup & Build Console SDK + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Load and Start Appwrite + run: | + docker compose build + docker compose up -d + docker compose exec appwrite sdks --platform=console --sdk=web --version=latest --git=no + sudo chown -R $USER:$USER ./app/sdks/console-web + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Build and Publish SDK + working-directory: ./app/sdks/console-web + run: | + npm install + npm run build + npx pkg-pr-new publish --comment=update diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 29b3bdd70d..3ab240c1b9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -151,10 +151,25 @@ jobs: sleep 30 - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug - env: - _APP_DATABASE_SHARED_TABLES: ${{ matrix.table_mode == 'Shared V1' || matrix.table_mode == 'Shared V2' && 'database_db_main' || '' }} - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.table_mode == 'Shared V1' && 'database_db_main' || '' }} + run: | + if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then + echo "Using shared tables V1" + export _APP_DATABASE_SHARED_TABLES=database_db_main + export _APP_DATABASE_SHARED_TABLES_V1=database_db_main + elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then + echo "Using shared tables V2" + export _APP_DATABASE_SHARED_TABLES=database_db_main + export _APP_DATABASE_SHARED_TABLES_V1= + else + echo "Using project tables" + export _APP_DATABASE_SHARED_TABLES= + export _APP_DATABASE_SHARED_TABLES_V1= + fi + + docker compose exec -T \ + -e _APP_DATABASE_SHARED_TABLES \ + -e _APP_DATABASE_SHARED_TABLES_V1 \ + appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug benchmarking: name: Benchmark diff --git a/.gitignore b/.gitignore index 5ae03e2a56..a68af84071 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ dev/yasd_init.php .phpunit.result.cache Makefile appwrite.json +/.zed/ diff --git a/Dockerfile b/Dockerfile index 07295f4afb..41810f5dc4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN composer install --ignore-platform-reqs --optimize-autoloader \ --no-plugins --no-scripts --prefer-dist \ `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi` -FROM appwrite/base:0.9.3 AS final +FROM appwrite/base:0.9.5 AS final LABEL maintainer="team@appwrite.io" diff --git a/app/config/function-templates.php b/app/config/function-templates.php index 762c33dd9a..fec9d2227f 100644 --- a/app/config/function-templates.php +++ b/app/config/function-templates.php @@ -11,7 +11,7 @@ const TEMPLATE_RUNTIMES = [ ], 'DART' => [ 'name' => 'dart', - 'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16', '2.16'] + 'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16'] ], 'GO' => [ 'name' => 'go', diff --git a/app/config/specs/open-api3-latest-client.json b/app/config/specs/open-api3-latest-client.json index e3cd909b4e..d948a101f2 100644 --- a/app/config/specs/open-api3-latest-client.json +++ b/app/config/specs/open-api3-latest-client.json @@ -2784,7 +2784,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 4e97fecc30..fb01509ecd 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -2795,7 +2795,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -20672,7 +20672,7 @@ }, "\/projects\/{projectId}\/auth\/memberships-privacy": { "patch": { - "summary": "Update project team sensitive attributes", + "summary": "Update project memberships privacy attributes", "operationId": "projectsUpdateMembershipsPrivacy", "tags": [ "projects" @@ -36077,17 +36077,17 @@ "description": "Whether or not to send session alert emails to users.", "x-example": true }, - "membershipsUserName": { + "authMembershipsUserName": { "type": "boolean", "description": "Whether or not to show user names in the teams membership response.", "x-example": true }, - "membershipsUserEmail": { + "authMembershipsUserEmail": { "type": "boolean", "description": "Whether or not to show user emails in the teams membership response.", "x-example": true }, - "membershipsMfa": { + "authMembershipsMfa": { "type": "boolean", "description": "Whether or not to show user MFA status in the teams membership response.", "x-example": true @@ -36297,9 +36297,9 @@ "authPersonalDataCheck", "authMockNumbers", "authSessionAlerts", - "membershipsUserName", - "membershipsUserEmail", - "membershipsMfa", + "authMembershipsUserName", + "authMembershipsUserEmail", + "authMembershipsMfa", "oAuthProviders", "platforms", "webhooks", diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index e84b751743..1504322456 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -2451,7 +2451,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", diff --git a/app/config/specs/swagger2-latest-client.json b/app/config/specs/swagger2-latest-client.json index b1b9ce8dca..3ac70ffe28 100644 --- a/app/config/specs/swagger2-latest-client.json +++ b/app/config/specs/swagger2-latest-client.json @@ -2922,7 +2922,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index c7b6bac2d4..25c0c289b3 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -2949,7 +2949,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -21138,7 +21138,7 @@ }, "\/projects\/{projectId}\/auth\/memberships-privacy": { "patch": { - "summary": "Update project team sensitive attributes", + "summary": "Update project memberships privacy attributes", "operationId": "projectsUpdateMembershipsPrivacy", "consumes": [ "application\/json" @@ -36591,17 +36591,17 @@ "description": "Whether or not to send session alert emails to users.", "x-example": true }, - "membershipsUserName": { + "authMembershipsUserName": { "type": "boolean", "description": "Whether or not to show user names in the teams membership response.", "x-example": true }, - "membershipsUserEmail": { + "authMembershipsUserEmail": { "type": "boolean", "description": "Whether or not to show user emails in the teams membership response.", "x-example": true }, - "membershipsMfa": { + "authMembershipsMfa": { "type": "boolean", "description": "Whether or not to show user MFA status in the teams membership response.", "x-example": true @@ -36815,9 +36815,9 @@ "authPersonalDataCheck", "authMockNumbers", "authSessionAlerts", - "membershipsUserName", - "membershipsUserEmail", - "membershipsMfa", + "authMembershipsUserName", + "authMembershipsUserEmail", + "authMembershipsMfa", "oAuthProviders", "platforms", "webhooks", diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 2c8e80c65e..68bcf268fb 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -2604,7 +2604,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 258d16cf46..a00324f3b7 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -20,7 +20,6 @@ use Utopia\Audit\Audit; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; @@ -399,19 +398,29 @@ function updateAttribute( } if (!empty($newKey) && $key !== $newKey) { - // Delete attribute and recreate since we can't modify IDs - $original = clone $attribute; - - $dbForProject->deleteDocument('attributes', $attribute->getId()); + $originalUid = $attribute->getId(); $attribute ->setAttribute('$id', ID::custom($db->getInternalId() . '_' . $collection->getInternalId() . '_' . $newKey)) ->setAttribute('key', $newKey); - try { - $attribute = $dbForProject->createDocument('attributes', $attribute); - } catch (DatabaseException|PDOException) { - $attribute = $dbForProject->createDocument('attributes', $original); + $dbForProject->updateDocument('attributes', $originalUid, $attribute); + + /** + * @var Document $index + */ + foreach ($collection->getAttribute('indexes') as $index) { + /** + * @var string[] $attributes + */ + $attributes = $index->getAttribute('attributes', []); + $found = \array_search($key, $attributes); + + if ($found !== false) { + $attributes[$found] = $newKey; + $index->setAttribute('attributes', $attributes); + $dbForProject->updateDocument('indexes', $index->getId(), $index); + } } } else { $attribute = $dbForProject->updateDocument('attributes', $db->getInternalId() . '_' . $collection->getInternalId() . '_' . $key, $attribute); @@ -2589,7 +2598,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $attributeStatus = $oldAttributes[$attributeIndex]['status']; $attributeType = $oldAttributes[$attributeIndex]['type']; - $attributeSize = $oldAttributes[$attributeIndex]['size']; $attributeArray = $oldAttributes[$attributeIndex]['array'] ?? false; if ($attributeType === Database::VAR_RELATIONSHIP) { @@ -2603,10 +2611,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $lengths[$i] = null; - if ($attributeType === Database::VAR_STRING) { - $lengths[$i] = $attributeSize; // set attribute size as index length only for strings - } - if ($attributeArray === true) { $lengths[$i] = Database::ARRAY_INDEX_LENGTH; $orders[$i] = null; diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 5b2fb49687..92d3103293 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -136,16 +136,6 @@ App::post('/v1/projects') $dsn = $databases[array_rand($databases)]; } - $dsns = [ - 'fra' => 'database_db_fra1_self_hosted_0_0', - 'syd' => 'database_db_syd1_self_hosted_0_0', - 'nyc' => 'database_db_nyc1_self_hosted_0_0', - ]; - - if (isset($dsns[$region])) { - $dsn = $dsns[$region]; - } - // TODO: Temporary until all projects are using shared tables. $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); @@ -215,86 +205,83 @@ App::post('/v1/projects') $sharedTablesV2 = !$projectTables && !$sharedTablesV1; $sharedTables = $sharedTablesV1 || $sharedTablesV2; - if ($sharedTables) { - $dbForProject - ->setSharedTables(true) - ->setTenant($sharedTablesV1 ? $project->getInternalId() : null) - ->setNamespace($dsn->getParam('namespace')); - } else { - $dbForProject - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - - $create = true; - - try { - $dbForProject->create(); - } catch (Duplicate) { - $create = false; - } - - if ($create || $projectTables) { - $audit = new Audit($dbForProject); - $audit->setup(); - - $abuse = new TimeLimit('', 0, 1, $dbForProject); - $abuse->setup(); - } - - if (!$create && $sharedTablesV1) { - $attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES); - $indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES); - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom('audit'), - '$permissions' => [Permission::create(Role::any())], - 'name' => 'audit', - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); - - $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES); - $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES); - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom('abuse'), - '$permissions' => [Permission::create(Role::any())], - 'name' => 'abuse', - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); - } - - if ($create || $sharedTablesV1) { - /** @var array $collections */ - $collections = Config::getParam('collections', [])['projects'] ?? []; - - foreach ($collections as $key => $collection) { - if (($collection['$collection'] ?? '') !== Database::METADATA) { - continue; + if (!$sharedTablesV2) { + if ($sharedTables) { + $dbForProject + ->setSharedTables(true) + ->setTenant($sharedTablesV1 ? $project->getInternalId() : null) + ->setNamespace($dsn->getParam('namespace')); + } else { + $dbForProject + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); } - $attributes = \array_map(function (array $attribute) { - return new Document($attribute); - }, $collection['attributes']); + $create = true; - $indexes = \array_map(function (array $index) { - return new Document($index); - }, $collection['indexes']); + try { + $dbForProject->create(); + } catch (Duplicate) { + $create = false; + } - try { - $dbForProject->createCollection($key, $attributes, $indexes); - } catch (Duplicate) { - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom($key), - '$permissions' => [Permission::create(Role::any())], - 'name' => $key, - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); + if ($create || $projectTables) { + $audit = new Audit($dbForProject); + $audit->setup(); + + $abuse = new TimeLimit('', 0, 1, $dbForProject); + $abuse->setup(); + } + + if (!$create && $sharedTablesV1) { + $attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES); + $indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES); + $dbForProject->createDocument(Database::METADATA, new Document([ + '$id' => ID::custom('audit'), + '$permissions' => [Permission::create(Role::any())], + 'name' => 'audit', + 'attributes' => $attributes, + 'indexes' => $indexes, + 'documentSecurity' => true + ])); + + $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES); + $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES); + $dbForProject->createDocument(Database::METADATA, new Document([ + '$id' => ID::custom('abuse'), + '$permissions' => [Permission::create(Role::any())], + 'name' => 'abuse', + 'attributes' => $attributes, + 'indexes' => $indexes, + 'documentSecurity' => true + ])); + } + + if ($create || $sharedTablesV1) { + /** @var array $collections */ + $collections = Config::getParam('collections', [])['projects'] ?? []; + + foreach ($collections as $key => $collection) { + if (($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } + + $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); + + try { + $dbForProject->createCollection($key, $attributes, $indexes); + } catch (Duplicate) { + $dbForProject->createDocument(Database::METADATA, new Document([ + '$id' => ID::custom($key), + '$permissions' => [Permission::create(Role::any())], + 'name' => $key, + 'attributes' => $attributes, + 'indexes' => $indexes, + 'documentSecurity' => true + ])); + } } } } diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index be055f0935..ba1858c5aa 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -805,8 +805,11 @@ App::get('/v1/teams/:teamId/memberships') }, $membershipsPrivacy); $memberships = array_map(function ($membership) use ($dbForProject, $team, $membershipsPrivacy) { + $user = !empty(array_filter($membershipsPrivacy)) + ? $dbForProject->getDocument('users', $membership->getAttribute('userId')) + : new Document(); + if ($membershipsPrivacy['mfa']) { - $user = $dbForProject->getDocument('users', $membership->getAttribute('userId')); $mfa = $user->getAttribute('mfa', false); if ($mfa) { @@ -888,9 +891,11 @@ App::get('/v1/teams/:teamId/memberships/:membershipId') return $privacy || $isPrivilegedUser || $isAppUser; }, $membershipsPrivacy); - if ($membershipsPrivacy['mfa']) { - $user = $dbForProject->getDocument('users', $membership->getAttribute('userId')); + $user = !empty(array_filter($membershipsPrivacy)) + ? $dbForProject->getDocument('users', $membership->getAttribute('userId')) + : new Document(); + if ($membershipsPrivacy['mfa']) { $mfa = $user->getAttribute('mfa', false); if ($mfa) { diff --git a/app/http.php b/app/http.php index 00c8eff21f..4bfacd4cac 100644 --- a/app/http.php +++ b/app/http.php @@ -12,6 +12,7 @@ use Swoole\Process; use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; @@ -90,7 +91,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg Console::success('[Setup] - Server database init started...'); try { - Console::success('[Setup] - Creating database: appwrite...'); + Console::success('[Setup] - Creating console database...'); $dbForConsole->create(); } catch (Duplicate) { Console::success('[Setup] - Skip: metadata table already exists'); @@ -117,34 +118,10 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg continue; } - Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); + Console::success('[Setup] - Creating console collection: ' . $collection['$id'] . '...'); - $attributes = []; - $indexes = []; - - foreach ($collection['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => ID::custom($attribute['$id']), - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } - - foreach ($collection['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => ID::custom($index['$id']), - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } + $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); $dbForConsole->createCollection($key, $attributes, $indexes); } @@ -179,36 +156,55 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg throw new Exception('Files collection is not configured.'); } - $attributes = []; - $indexes = []; - - foreach ($files['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => ID::custom($attribute['$id']), - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } - - foreach ($files['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => ID::custom($index['$id']), - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } + $attributes = \array_map(fn ($attribute) => new Document($attribute), $files['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $files['indexes']); $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } + $projectCollections = $collections['projects']; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); + $sharedTablesV2 = \array_diff($sharedTables, $sharedTablesV1); + + $cache = $app->getResource('cache'); + + foreach ($sharedTablesV2 as $hostname) { + $adapter = $pools + ->get($hostname) + ->pop() + ->getResource(); + + $dbForProject = (new Database($adapter, $cache)) + ->setDatabase('appwrite') + ->setSharedTables(true) + ->setTenant(null) + ->setNamespace(System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '')); + + try { + Console::success('[Setup] - Creating project database: ' . $hostname . '...'); + $dbForProject->create(); + } catch (Duplicate) { + Console::success('[Setup] - Skip: metadata table already exists'); + } + + foreach ($projectCollections as $key => $collection) { + if (($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } + if (!$dbForProject->getCollection($key)->isEmpty()) { + continue; + } + + $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); + + Console::success('[Setup] - Creating project collection: ' . $collection['$id'] . '...'); + + $dbForProject->createCollection($key, $attributes, $indexes); + } + } + $pools->reclaim(); Console::success('[Setup] - Server database init completed...'); diff --git a/app/init.php b/app/init.php index f47176caae..97efbcf134 100644 --- a/app/init.php +++ b/app/init.php @@ -1405,7 +1405,7 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForPltroject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { +App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } @@ -1430,13 +1430,6 @@ App::setResource('dbForPltroject', function (Group $pools, Database $dbForConsol ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); if (\in_array($dsn->getHost(), $sharedTables)) { diff --git a/app/realtime.php b/app/realtime.php index fe1241eb21..cfb9bf9a16 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -29,6 +29,7 @@ use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\System\System; +use Utopia\Telemetry\Adapter\None as NoTelemetry; use Utopia\WebSocket\Adapter; use Utopia\WebSocket\Server; @@ -144,6 +145,13 @@ if (!function_exists('getRealtime')) { } } +if (!function_exists('getTelemetry')) { + function getTelemetry(int $workerId): Utopia\Telemetry\Adapter + { + return new NoTelemetry(); + } +} + $realtime = getRealtime(); /** @@ -276,6 +284,12 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $logError) { Console::success('Worker ' . $workerId . ' started successfully'); + $telemetry = getTelemetry($workerId); + $register->set('telemetry', fn () => $telemetry); + $register->set('telemetry.connectionCounter', fn () => $telemetry->createUpDownCounter('realtime.server.open_connections')); + $register->set('telemetry.connectionCreatedCounter', fn () => $telemetry->createCounter('realtime.server.connection.created')); + $register->set('telemetry.messageSentCounter', fn () => $telemetry->createCounter('realtime.server.message.sent')); + $attempts = 0; $start = time(); @@ -418,6 +432,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, ); if (($num = count($receivers)) > 0) { + $register->get('telemetry.messageSentCounter')->add($num); $stats->incr($event['project'], 'messages', $num); } }); @@ -521,6 +536,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ] ])); + $register->get('telemetry.connectionCounter')->add(1); + $register->get('telemetry.connectionCreatedCounter')->add(1); + $stats->set($project->getId(), [ 'projectId' => $project->getId(), 'teamId' => $project->getAttribute('teamId') @@ -594,9 +612,12 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } switch ($message['type']) { - /** - * This type is used to authenticate. - */ + case 'ping': + $server->send([$connection], json_encode([ + 'type' => 'pong' + ])); + + break; case 'authentication': if (!array_key_exists('session', $message['data'])) { throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.'); @@ -654,12 +675,14 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } }); -$server->onClose(function (int $connection) use ($realtime, $stats) { +$server->onClose(function (int $connection) use ($realtime, $stats, $register) { if (array_key_exists($connection, $realtime->connections)) { $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal'); } $realtime->unsubscribe($connection); + $register->get('telemetry.connectionCounter')->add(-1); + Console::info('Connection close: ' . $connection); }); diff --git a/app/worker.php b/app/worker.php index 25defba3f8..7899c1d5bc 100644 --- a/app/worker.php +++ b/app/worker.php @@ -283,6 +283,50 @@ Server::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); +Server::setResource('logError', function (Registry $register, Document $project) { + return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { + $logger = $register->get('logger'); + + if ($logger) { + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace($namespace); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + $log->addTag('projectId', $project->getId() ?? ''); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + + + foreach ($extras as $key => $value) { + $log->addExtra($key, $value); + } + + $log->setAction($action); + + $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + try { + $responseCode = $logger->addLog($log); + Console::info('Error log pushed with status code: ' . $responseCode); + } catch (Throwable $th) { + Console::error('Error pushing log: ' . $th->getMessage()); + } + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; +}, ['register', 'project']); $pools = $register->get('pools'); $platform = new Appwrite(); diff --git a/composer.json b/composer.json index c11f78757b..e3fbcabc83 100644 --- a/composer.json +++ b/composer.json @@ -70,6 +70,7 @@ "utopia-php/storage": "0.18.*", "utopia-php/swoole": "0.8.*", "utopia-php/system": "0.9.*", + "utopia-php/telemetry": "0.1.*", "utopia-php/vcs": "0.8.*", "utopia-php/websocket": "0.1.*", "matomo/device-detector": "6.1.*", @@ -96,6 +97,10 @@ "config": { "platform": { "php": "8.3" + }, + "allow-plugins": { + "php-http/discovery": false, + "tbachert/spi": false } } } diff --git a/docker-compose.yml b/docker-compose.yml index 0f3f05847c..bae2cc7811 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -194,6 +194,7 @@ services: - _APP_EXPERIMENT_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 + - _APP_DATABASE_SHARED_NAMESPACE appwrite-console: <<: *x-logging @@ -383,6 +384,7 @@ services: - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_DATABASE_SHARED_TABLES + - _APP_DATABASE_SHARED_TABLES_V1 appwrite-worker-databases: entrypoint: worker-databases diff --git a/docs/references/account/create-token-magic-url.md b/docs/references/account/create-token-magic-url.md index 6ebe4154b8..99ad6dba5e 100644 --- a/docs/references/account/create-token-magic-url.md +++ b/docs/references/account/create-token-magic-url.md @@ -1,3 +1,3 @@ -Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST /v1/account/sessions/token](https://appwrite.io/docs/references/cloud/client-web/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default. +Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST /v1/account/sessions/token](https://appwrite.io/docs/references/cloud/client-web/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. A user is limited to 10 active sessions at a time by default. [Learn more about session limits](https://appwrite.io/docs/authentication-security#limits). diff --git a/src/Appwrite/Platform/Tasks/SDKs.php b/src/Appwrite/Platform/Tasks/SDKs.php index 65d2f7717c..6e37f270a4 100644 --- a/src/Appwrite/Platform/Tasks/SDKs.php +++ b/src/Appwrite/Platform/Tasks/SDKs.php @@ -25,6 +25,9 @@ use Appwrite\Spec\Swagger2; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Platform\Action; +use Utopia\Validator\Nullable; +use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; class SDKs extends Action { @@ -37,23 +40,35 @@ class SDKs extends Action { $this ->desc('Generate Appwrite SDKs') - ->callback(fn () => $this->action()); + ->param('platform', null, new Nullable(new Text(256)), 'Selected Platform', optional: true) + ->param('sdk', null, new Nullable(new Text(256)), 'Selected SDK', optional:true) + ->param('version', null, new Nullable(new Text(256)), 'Selected SDK', optional:true) + ->param('git', null, new Nullable(new WhiteList(['yes', 'no'])), 'Should we use git push?', optional: true) + ->param('production', null, new Nullable(new WhiteList(['yes', 'no'])), 'Should we push to production?', optional:true) + ->param('message', null, new Nullable(new Text(256)), 'Commit Message', optional:true) + ->callback([$this, 'action']); } - public function action(): void + public function action(?string $selectedPlatform, ?string $selectedSDK, ?string $version, ?string $git, ?string $production, ?string $message) { - $platforms = Config::getParam('platforms'); - $selectedPlatform = Console::confirm('Choose Platform ("' . APP_PLATFORM_CLIENT . '", "' . APP_PLATFORM_SERVER . '", "' . APP_PLATFORM_CONSOLE . '" or "*" for all):'); - $selectedSDK = \strtolower(Console::confirm('Choose SDK ("*" for all):')); - $version = Console::confirm('Choose an Appwrite version'); - $git = (Console::confirm('Should we use git push? (yes/no)') == 'yes'); - $production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false; - $message = ($git) ? Console::confirm('Please enter your commit message:') : ''; + $selectedPlatform ??= Console::confirm('Choose Platform ("' . APP_PLATFORM_CLIENT . '", "' . APP_PLATFORM_SERVER . '", "' . APP_PLATFORM_CONSOLE . '" or "*" for all):'); + $selectedSDK ??= \strtolower(Console::confirm('Choose SDK ("*" for all):')); + $version ??= Console::confirm('Choose an Appwrite version'); + + $git ??= Console::confirm('Should we use git push? (yes/no)'); + $git = $git === 'yes'; + + if ($git) { + $production ??= Console::confirm('Type "Appwrite" to push code to production git repos'); + $production = $production === 'Appwrite'; + $message ??= Console::confirm('Please enter your commit message:'); + } if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', '1.4.x', '1.5.x', '1.6.x', 'latest'])) { throw new \Exception('Unknown version given'); } + $platforms = Config::getParam('platforms'); foreach ($platforms as $key => $platform) { if ($selectedPlatform !== $key && $selectedPlatform !== '*') { continue; diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index fe81114ea0..763967bb56 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -354,11 +354,9 @@ class Databases extends Action } } finally { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); - $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); - $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); } } } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 700a480065..6c11116a4c 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -494,31 +494,31 @@ class Deletes extends Action ]; $limit = \count($projectCollectionIds) + 25; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); + + $projectTables = !\in_array($dsn->getHost(), $sharedTables); + $sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1); + $sharedTablesV2 = !$projectTables && !$sharedTablesV1; + $sharedTables = $sharedTablesV1 || $sharedTablesV2; while (true) { $collections = $dbForProject->listCollections($limit); foreach ($collections as $collection) { - if (\in_array($dsn->getHost(), $sharedTables) || !\in_array($collection->getId(), $projectCollectionIds)) { - try { + try { + if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); - } catch (Throwable $e) { - Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); - - /** - * Ignore junction tables; - */ - if (!preg_match('/^_\d+_\d+$/', $collection->getId())) { - throw $e; - } + } else { + $this->deleteByGroup($collection->getId(), [], database: $dbForProject); } - } else { - $this->deleteByGroup($collection->getId(), [], database: $dbForProject); + } catch (Throwable $e) { + Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); } } - if (\in_array($dsn->getHost(), $sharedTables)) { + if ($sharedTables) { $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections); if (empty(\array_diff($collectionsIds, $projectCollectionIds))) { @@ -572,10 +572,17 @@ class Deletes extends Action ], $dbForConsole); // Delete metadata table - if (\in_array($dsn->getHost(), $sharedTables)) { - $dbForProject->deleteCollection('_metadata'); - } else { - $this->deleteByGroup('_metadata', [], $dbForProject); + if ($projectTables) { + $dbForProject->deleteCollection(Database::METADATA); + } elseif ($sharedTablesV1) { + $this->deleteByGroup(Database::METADATA, [], $dbForProject); + } elseif ($sharedTablesV2) { + $queries = \array_map( + fn ($id) => Query::notEqual('$id', $id), + $projectCollectionIds + ); + + $this->deleteByGroup(Database::METADATA, $queries, $dbForProject); } // Delete all storage directories diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 4b1926ed26..fdd885effa 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -16,7 +16,6 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; -use Utopia\Logger\Log; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; @@ -37,6 +36,8 @@ class Migrations extends Action protected Document $project; + protected $logError; + public static function getName(): string { return 'migrations'; @@ -52,14 +53,14 @@ class Migrations extends Action ->inject('message') ->inject('dbForProject') ->inject('dbForConsole') - ->inject('log') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log)); + ->inject('logError') + ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, callable $logError) => $this->action($message, $dbForProject, $dbForConsole, $logError)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void + public function action(Message $message, Database $dbForProject, Database $dbForConsole, callable $logError): void { $payload = $message->getPayload() ?? []; @@ -78,6 +79,7 @@ class Migrations extends Action $this->dbForProject = $dbForProject; $this->dbForConsole = $dbForConsole; $this->project = $project; + $this->logError = $logError; /** * Handle Event execution. @@ -86,10 +88,7 @@ class Migrations extends Action return; } - $log->addTag('migrationId', $migration->getId()); - $log->addTag('projectId', $project->getId()); - - $this->processMigration($migration, $log); + $this->processMigration($migration); } /** @@ -259,7 +258,7 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration, Log $log): void + protected function processMigration(Document $migration): void { $project = $this->project; $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); @@ -285,8 +284,6 @@ class Migrations extends Action $migration->setAttribute('status', 'processing'); $this->updateMigrationDocument($migration, $projectDocument); - $log->addTag('type', $migration->getAttribute('source')); - $source = $this->processSource($migration); $destination = $this->processDestination($migration, $tempAPIKey->getAttribute('secret')); @@ -324,7 +321,6 @@ class Migrations extends Action $errorMessages = []; foreach ($sourceErrors as $error) { - /** @var $sourceErrors $error */ $message = "Error occurred while fetching '{$error->getResourceName()}:{$error->getResourceId()}' from source with message: '{$error->getMessage()}'"; if ($error->getPrevious()) { $message .= " Message: ".$error->getPrevious()->getMessage() . " File: ".$error->getPrevious()->getFile() . " Line: ".$error->getPrevious()->getLine(); @@ -344,7 +340,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $log->addExtra('migrationErrors', json_encode($errorMessages)); $this->updateMigrationDocument($migration, $projectDocument); return; @@ -359,7 +354,6 @@ class Migrations extends Action if (! $migration->isEmpty()) { $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - $migration->setAttribute('errors', [$th->getMessage()]); return; } @@ -379,7 +373,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $log->addTag('migrationErrors', json_encode($errorMessages)); } } finally { if (! $tempAPIKey->isEmpty()) { @@ -394,7 +387,23 @@ class Migrations extends Action $destination->error(); $source->error(); - throw new Exception('Migration failed'); + foreach ($source->getErrors() as $error) { + call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + 'migrationId' => $migration->getId() ?? '', + 'source' => $migration->getAttribute('source') ?? '', + 'resourceName' => $error->getResourceName(), + 'resourceGroup' => $error->getResourceGroup() + ]); + } + + foreach ($destination->getErrors() as $error) { + call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + 'migrationId' => $migration->getId() ?? '', + 'source' => $migration->getAttribute('source') ?? '', + 'resourceName' => $error->getResourceName(), + 'resourceGroup' => $error->getResourceGroup() + ]); + } } if ($migration->getAttribute('status', '') === 'completed') { diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index f7430ec70e..65df0cb694 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -323,6 +323,7 @@ class OpenAPI3 extends Format case 'Utopia\Validator\JSON': case 'Utopia\Validator\Mock': case 'Utopia\Validator\Assoc': + case 'Appwrite\Functions\Validator\Payload': $param['default'] = (empty($param['default'])) ? new \stdClass() : $param['default']; $node['schema']['type'] = 'object'; $node['schema']['x-example'] = '{}'; diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index f2e324c71f..66a540a4d4 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -349,6 +349,7 @@ class Swagger2 extends Format case 'Utopia\Validator\JSON': case 'Utopia\Validator\Mock': case 'Utopia\Validator\Assoc': + case 'Appwrite\Functions\Validator\Payload': $node['type'] = 'object'; $node['default'] = (empty($param['default'])) ? new \stdClass() : $param['default']; $node['x-example'] = '{}'; diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 7b0847126c..326443afcf 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -3727,11 +3727,11 @@ class ProjectsConsoleClientTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'teamId' => ID::unique(), - 'name' => 'Amating Team', + 'name' => 'Amazing Team', ]); $this->assertEquals(201, $team['headers']['status-code']); - $this->assertEquals('Amating Team', $team['body']['name']); + $this->assertEquals('Amazing Team', $team['body']['name']); $this->assertNotEmpty($team['body']['$id']); $teamId = $team['body']['$id']; @@ -3794,6 +3794,115 @@ class ProjectsConsoleClientTest extends Scope return $data; } + public function testDeleteSharedProject(): void + { + $team = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'teamId' => ID::unique(), + 'name' => 'Amazing Team', + ]); + + $teamId = $team['body']['$id']; + + // Ensure deleting one project does not affect another project + $project1 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project 1', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project 2', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + $project1Id = $project1['body']['$id']; + $project2Id = $project2['body']['$id']; + + // Create user in each project + $key1 = $this->client->call(Client::METHOD_POST, '/projects/' . $project1Id . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['users.read', 'users.write'], + ]); + + $user1 = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1Id, + 'x-appwrite-key' => $key1['body']['secret'], + ], [ + 'userId' => ID::unique(), + 'email' => 'test1@appwrite.io', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user1['headers']['status-code']); + + $key2 = $this->client->call(Client::METHOD_POST, '/projects/' . $project2Id . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['users.read', 'users.write'], + ]); + + $user2 = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2Id, + 'x-appwrite-key' => $key2['body']['secret'], + ], [ + 'userId' => ID::unique(), + 'email' => 'test2@appwrite.io', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user2['headers']['status-code']); + + // Delete project 1 + $project1 = $this->client->call(Client::METHOD_DELETE, '/projects/' . $project1Id, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(204, $project1['headers']['status-code']); + + \sleep(3); + + // Ensure project 2 user is still there + $user2 = $this->client->call(Client::METHOD_GET, '/users/' . $user2['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2Id, + 'x-appwrite-key' => $key2['body']['secret'], + ]); + + $this->assertEquals(200, $user2['headers']['status-code']); + + // Create another user in project 2 in case read hits stale cache + $user3 = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2Id, + 'x-appwrite-key' => $key2['body']['secret'], + ], [ + 'userId' => ID::unique(), + 'email' => 'test3@appwrite.io' + ]); + + $this->assertEquals(201, $user3['headers']['status-code']); + } + /** * @depends testCreateProject */ diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index 616f309fd2..dda524fc7c 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -111,6 +111,30 @@ class RealtimeCustomClientTest extends Scope $client->close(); } + public function testPingPong() + { + $client = $this->getWebsocket(['files'], [ + 'origin' => 'http://localhost' + ]); + $response = json_decode($client->receive(), true); + + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('connected', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertCount(1, $response['data']['channels']); + $this->assertContains('files', $response['data']['channels']); + + $client->send(\json_encode([ + 'type' => 'ping' + ])); + + $response = json_decode($client->receive(), true); + $this->assertEquals('pong', $response['type']); + + $client->close(); + } + public function testManualAuthentication() { $user = $this->getUser(); diff --git a/tests/e2e/Services/Teams/TeamsCustomClientTest.php b/tests/e2e/Services/Teams/TeamsCustomClientTest.php index 03cb1983f1..7286bb0827 100644 --- a/tests/e2e/Services/Teams/TeamsCustomClientTest.php +++ b/tests/e2e/Services/Teams/TeamsCustomClientTest.php @@ -83,6 +83,38 @@ class TeamsCustomClientTest extends Scope $this->assertNotEmpty($response['body']['memberships'][0]['userName']); $this->assertNotEmpty($response['body']['memberships'][0]['userEmail']); $this->assertArrayHasKey('mfa', $response['body']['memberships'][0]); + + /** + * Update project settings to show only MFA + */ + $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/memberships-privacy', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ]), [ + 'userName' => false, + 'userEmail' => false, + 'mfa' => true, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + /** + * Test that sensitive fields are not shown + */ + $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsInt($response['body']['total']); + $this->assertNotEmpty($response['body']['memberships'][0]['$id']); + + // Assert that sensitive fields are present + $this->assertEmpty($response['body']['memberships'][0]['userName']); + $this->assertEmpty($response['body']['memberships'][0]['userEmail']); + $this->assertArrayHasKey('mfa', $response['body']['memberships'][0]); } /**