From a0cd65b94bcc4f97c3a8c2910a62a680deb7e7cf Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 16 Aug 2023 16:01:56 +0100 Subject: [PATCH 1/9] Fix Migrations Stability --- app/controllers/api/migrations.php | 211 ++++++++++++++------------ app/controllers/general.php | 4 +- app/controllers/web/console.php | 2 +- app/init.php | 1 + src/Appwrite/Auth/OAuth2/Firebase.php | 51 ++++++- 5 files changed, 164 insertions(+), 105 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 8d961a0450..b55bf59af0 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -34,7 +34,7 @@ App::post('/v1/migrations/appwrite') ->groups(['api', 'migrations']) ->desc('Migrate Appwrite Data') ->label('scope', 'migrations.write') - ->label('event', 'migrations.create') + ->label('event', 'migrations.[migrationId].create') ->label('audits.event', 'migration.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'migrations') @@ -88,7 +88,7 @@ App::post('/v1/migrations/firebase/oauth') ->groups(['api', 'migrations']) ->desc('Migrate Firebase Data (OAuth)') ->label('scope', 'migrations.write') - ->label('event', 'migrations.create') + ->label('event', 'migrations.[migrationId].create') ->label('audits.event', 'migration.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'migrations') @@ -146,12 +146,13 @@ App::post('/v1/migrations/firebase/oauth') $dbForConsole->updateDocument('identities', $identity->getId(), $identity); } - if ($identity->getAttribute('secret')) { - $serviceAccount = $identity->getAttribute('secret'); + if ($identity->getAttribute('secrets')) { + $serviceAccount = $identity->getAttribute('secrets'); } else { + $firebase->cleanupServiceAccounts($accessToken, $projectId); $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId); $identity = $identity - ->setAttribute('secret', $serviceAccount); + ->setAttribute('secrets', json_encode($serviceAccount)); $dbForConsole->updateDocument('identities', $identity->getId(), $identity); } @@ -189,7 +190,7 @@ App::post('/v1/migrations/firebase') ->groups(['api', 'migrations']) ->desc('Migrate Firebase Data (Service Account)') ->label('scope', 'migrations.write') - ->label('event', 'migrations.create') + ->label('event', 'migrations.[migrationId].create') ->label('audits.event', 'migration.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'migrations') @@ -239,7 +240,7 @@ App::post('/v1/migrations/supabase') ->groups(['api', 'migrations']) ->desc('Migrate Supabase Data') ->label('scope', 'migrations.write') - ->label('event', 'migrations.create') + ->label('event', 'migrations.[migrationId].create') ->label('audits.event', 'migration.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'migrations') @@ -299,7 +300,7 @@ App::post('/v1/migrations/nhost') ->groups(['api', 'migrations']) ->desc('Migrate NHost Data') ->label('scope', 'migrations.write') - ->label('event', 'migrations.create') + ->label('event', 'migrations.[migrationId].create') ->label('audits.event', 'migration.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'migrations') @@ -309,7 +310,7 @@ App::post('/v1/migrations/nhost') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_MIGRATION) ->param('resources', [], new ArrayList(new WhiteList(NHost::getSupportedResources())), 'List of resources to migrate') - ->param('subdomain', '', new URL(), 'Source\'s Subdomain') + ->param('subdomain', '', new Text(512), 'Source\'s Subdomain') ->param('region', '', new Text(512), 'Source\'s Region') ->param('adminSecret', '', new Text(512), 'Source\'s Admin Secret') ->param('database', '', new Text(512), 'Source\'s Database Name') @@ -501,67 +502,73 @@ App::get('/v1/migrations/firebase/report/oauth') ->inject('user') ->inject('dbForConsole') ->action(function (array $resources, string $projectId, Response $response, Request $request, Document $user, Database $dbForConsole) { - try { - $firebase = new OAuth2Firebase( - App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), - App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), - $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' - ); + $firebase = new OAuth2Firebase( + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), + $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' + ); - $identity = $dbForConsole->findOne('identities', [ - Query::equal('provider', ['firebase']), - Query::equal('userInternalId', [$user->getInternalId()]), - ]); - if ($identity === false || $identity->isEmpty()) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } + $identity = $dbForConsole->findOne('identities', [ + Query::equal('provider', ['firebase']), + Query::equal('userInternalId', [$user->getInternalId()]), + ]); - $accessToken = $identity->getAttribute('providerAccessToken'); - $refreshToken = $identity->getAttribute('providerRefreshToken'); - $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); - - $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); - if ($isExpired) { - $firebase->refreshTokens($refreshToken); - - $accessToken = $firebase->getAccessToken(''); - $refreshToken = $firebase->getRefreshToken(''); - - $verificationId = $firebase->getUserID($accessToken); - - if (empty($verificationId)) { - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); - } - - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); - - $dbForConsole->updateDocument('identities', $identity->getId(), $identity); - } - - // Get Service Account - if ($identity->getAttribute('secret')) { - $serviceAccount = $identity->getAttribute('secret'); - } else { - $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId); - $identity = $identity - ->setAttribute('secret', $serviceAccount); - - $dbForConsole->updateDocument('identities', $identity->getId(), $identity); - } - - $firebase = new Firebase(array_merge($serviceAccount, ['project_id' => $projectId])); - - $report = $firebase->report($resources); - - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT); - } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + if ($identity === false || $identity->isEmpty()) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); } + + $accessToken = $identity->getAttribute('providerAccessToken'); + $refreshToken = $identity->getAttribute('providerRefreshToken'); + $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); + + if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); + } + + if (App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); + } + + $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); + if ($isExpired) { + $firebase->refreshTokens($refreshToken); + + $accessToken = $firebase->getAccessToken(''); + $refreshToken = $firebase->getRefreshToken(''); + + $verificationId = $firebase->getUserID($accessToken); + + if (empty($verificationId)) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); + } + + $identity = $identity + ->setAttribute('providerAccessToken', $accessToken) + ->setAttribute('providerRefreshToken', $refreshToken) + ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } + + // Get Service Account + if ($identity->getAttribute('secrets')) { + $serviceAccount = $identity->getAttribute('secrets'); + } else { + $firebase->cleanupServiceAccounts($accessToken, $projectId); + $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId); + $identity = $identity + ->setAttribute('secrets', json_encode($serviceAccount)); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } + + $firebase = new Firebase(array_merge($serviceAccount, ['project_id' => $projectId])); + + $report = $firebase->report($resources); + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT); }); App::get('/v1/migrations/firebase/connect') @@ -745,6 +752,7 @@ App::get('/v1/migrations/firebase/projects') Query::equal('provider', ['firebase']), Query::equal('userInternalId', [$user->getInternalId()]), ]); + if ($identity === false || $identity->isEmpty()) { throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); } @@ -754,46 +762,50 @@ App::get('/v1/migrations/firebase/projects') $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) { - throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Not authenticated with Firebase'); + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); } if (App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') { - throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Missing Google OAuth credentials'); + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); } - $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); - if ($isExpired) { - try { - $firebase->refreshTokens($refreshToken); - } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Failed to refresh Firebase access token'); + try { + $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); + if ($isExpired) { + try { + $firebase->refreshTokens($refreshToken); + } catch (\Exception $e) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); + } + + $accessToken = $firebase->getAccessToken(''); + $refreshToken = $firebase->getRefreshToken(''); + + $verificationId = $firebase->getUserID($accessToken); + + if (empty($verificationId)) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); + } + + $identity = $identity + ->setAttribute('providerAccessToken', $accessToken) + ->setAttribute('providerRefreshToken', $refreshToken) + ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); } - $accessToken = $firebase->getAccessToken(''); - $refreshToken = $firebase->getRefreshToken(''); + $projects = $firebase->getProjects($accessToken); - $verificationId = $firebase->getUserID($accessToken); - - if (empty($verificationId)) { - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); + $output = []; + foreach ($projects as $project) { + $output[] = [ + 'displayName' => $project['displayName'], + 'projectId' => $project['projectId'], + ]; } - - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); - - $dbForConsole->updateDocument('identities', $identity->getId(), $identity); - } - - $projects = $firebase->getProjects($accessToken); - - $output = []; - foreach ($projects as $project) { - $output[] = [ - 'displayName' => $project['displayName'], - 'projectId' => $project['projectId'], - ]; + } catch (\Exception $e) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); } $response->dynamic(new Document([ @@ -955,9 +967,8 @@ App::delete('/v1/migrations/:migrationId') ->param('migrationId', '', new UID(), 'Migration ID.') ->inject('response') ->inject('dbForProject') - ->inject('deletes') ->inject('events') - ->action(function (string $migrationId, Response $response, Database $dbForProject, Delete $deletes, Event $events) { + ->action(function (string $migrationId, Response $response, Database $dbForProject, Event $events) { $migration = $dbForProject->getDocument('migrations', $migrationId); if ($migration->isEmpty()) { diff --git a/app/controllers/general.php b/app/controllers/general.php index 11a83933ea..7a0cbda9f0 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -191,7 +191,7 @@ App::init() $host = $request->getHostname() ?? ''; $mainDomain = App::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain && $host !== 'localhost' && $host !== 'appwrite') { + if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) { if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) { return; } @@ -489,7 +489,7 @@ App::options() $host = $request->getHostname() ?? ''; $mainDomain = App::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain && $host !== 'localhost' && $host !== 'appwrite') { + if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) { if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) { return; } diff --git a/app/controllers/web/console.php b/app/controllers/web/console.php index 849ea85882..97e9a3932b 100644 --- a/app/controllers/web/console.php +++ b/app/controllers/web/console.php @@ -34,7 +34,7 @@ App::get('/console/*') // Serve static files (console) only for main domain $host = $request->getHostname() ?? ''; $mainDomain = App::getEnv('_APP_DOMAIN', ''); - if ($host !== $mainDomain && $host !== 'localhost' && $host !== 'appwrite') { + if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) { throw new Exception(Exception::GENERAL_ROUTE_NOT_FOUND); } diff --git a/app/init.php b/app/init.php index 646a9d1cce..79390b8290 100644 --- a/app/init.php +++ b/app/init.php @@ -135,6 +135,7 @@ const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244'; const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; +const APP_INTERNAL_LOOPBACK_HOST = 'appwrite'; // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; diff --git a/src/Appwrite/Auth/OAuth2/Firebase.php b/src/Appwrite/Auth/OAuth2/Firebase.php index 0a813881be..9c9c4acd4c 100644 --- a/src/Appwrite/Auth/OAuth2/Firebase.php +++ b/src/Appwrite/Auth/OAuth2/Firebase.php @@ -231,9 +231,22 @@ class Firebase extends OAuth2 ])); } + function generateRandomString($length = 10) + { + $characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; + $charactersLength = strlen($characters); + $randomString = ''; + for ($i = 0; $i < $length; $i++) { + $randomString .= $characters[random_int(0, $charactersLength - 1)]; + } + return $randomString; + } + public function createServiceAccount(string $accessToken, string $projectId): array { // Create Service Account + $uid = $this->generateRandomString(); + $response = $this->request( 'POST', 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts', @@ -242,9 +255,9 @@ class Firebase extends OAuth2 'Content-Type: application/json' ], json_encode([ - 'accountId' => 'appwrite-migrations', + 'accountId' => 'appwrite-' . $uid, 'serviceAccount' => [ - 'displayName' => 'Appwrite Migrations' + 'displayName' => 'Appwrite Migrations ' . $uid ] ]) ); @@ -267,4 +280,38 @@ class Firebase extends OAuth2 return json_decode(base64_decode($responseKey['privateKeyData']), true); } + + public function cleanupServiceAccounts(string $accessToken, string $projectId) + { + // List Service Accounts + $response = $this->request( + 'GET', + 'https://iam.googleapis.com/v1/projects/'.$projectId.'/serviceAccounts', + [ + 'Authorization: Bearer ' . \urlencode($accessToken), + 'Content-Type: application/json' + ] + ); + + $response = json_decode($response, true); + + if (empty($response['accounts'])) { + return false; + } + + foreach ($response['accounts'] as $account) { + if (strpos($account['email'], 'appwrite-') !== false) { + $this->request( + 'DELETE', + 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $account['email'], + [ + 'Authorization: Bearer ' . \urlencode($accessToken), + 'Content-Type: application/json' + ] + ); + } + } + + return true; + } } From 47b10fa82bc7724f3d573ee9396291d17f8a0b66 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 16 Aug 2023 18:17:20 +0100 Subject: [PATCH 2/9] Implement IAM Custom Roles --- app/controllers/api/migrations.php | 2 +- composer.json | 6 +- composer.lock | 42 +++++------ src/Appwrite/Auth/OAuth2/Firebase.php | 102 ++++++++++++++++++++++---- 4 files changed, 114 insertions(+), 38 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index b55bf59af0..870c5d7d80 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -562,7 +562,7 @@ App::get('/v1/migrations/firebase/report/oauth') $dbForConsole->updateDocument('identities', $identity->getId(), $identity); } - $firebase = new Firebase(array_merge($serviceAccount, ['project_id' => $projectId])); + $firebase = new Firebase($serviceAccount); $report = $firebase->report($resources); diff --git a/composer.json b/composer.json index 85813760a6..e2f24388d8 100644 --- a/composer.json +++ b/composer.json @@ -76,12 +76,16 @@ "webonyx/graphql-php": "14.11.*", "slickdeals/statsd": "3.1.0", "league/csv": "9.7.1", - "utopia-php/migration": "^0.2.0" + "utopia-php/migration": "^0.3.0" }, "repositories": [ { "url": "https://github.com/appwrite/runtimes.git", "type": "git" + }, + { + "url": "https://github.com/utopia-php/vcs.git", + "type": "git" } ], "require-dev": { diff --git a/composer.lock b/composer.lock index dd63ef4016..df4926d764 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a7de7a77dd4c58b74bc8fe9c0387979e", + "content-hash": "b87e28f6f096af1fd3b1ddee62fe2f13", "packages": [ { "name": "adhocore/jwt", @@ -1963,16 +1963,16 @@ }, { "name": "utopia-php/migration", - "version": "0.2.0", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "9dc59bbe0d126e20434580a5aa7cae5793bab024" + "reference": "8a1d4de19002e4c8eabf368edff6b653b6722880" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/9dc59bbe0d126e20434580a5aa7cae5793bab024", - "reference": "9dc59bbe0d126e20434580a5aa7cae5793bab024", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/8a1d4de19002e4c8eabf368edff6b653b6722880", + "reference": "8a1d4de19002e4c8eabf368edff6b653b6722880", "shasum": "" }, "require": { @@ -2015,9 +2015,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.2.0" + "source": "https://github.com/utopia-php/migration/tree/0.3.0" }, - "time": "2023-08-09T16:28:43+00:00" + "time": "2023-08-16T11:57:13+00:00" }, { "name": "utopia-php/mongo", @@ -2844,7 +2844,7 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/master" + "source": "https://github.com/appwrite/sdk-generator/tree/0.34.0" }, "time": "2023-07-25T01:15:31+00:00" }, @@ -3150,16 +3150,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.16.0", + "version": "v4.17.1", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "19526a33fb561ef417e822e85f08a00db4059c17" + "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/19526a33fb561ef417e822e85f08a00db4059c17", - "reference": "19526a33fb561ef417e822e85f08a00db4059c17", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", + "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", "shasum": "" }, "require": { @@ -3200,9 +3200,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.16.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1" }, - "time": "2023-06-25T14:52:30+00:00" + "time": "2023-08-13T19:53:39+00:00" }, { "name": "phar-io/manifest", @@ -3427,16 +3427,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.7.2", + "version": "1.7.3", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d" + "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/b2fe4d22a5426f38e014855322200b97b5362c0d", - "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419", + "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419", "shasum": "" }, "require": { @@ -3479,9 +3479,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.2" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.3" }, - "time": "2023-05-30T18:13:47+00:00" + "time": "2023-08-12T11:01:26+00:00" }, { "name": "phpspec/prophecy", @@ -5444,5 +5444,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } diff --git a/src/Appwrite/Auth/OAuth2/Firebase.php b/src/Appwrite/Auth/OAuth2/Firebase.php index 9c9c4acd4c..93078e5f1d 100644 --- a/src/Appwrite/Auth/OAuth2/Firebase.php +++ b/src/Appwrite/Auth/OAuth2/Firebase.php @@ -27,6 +27,38 @@ class Firebase extends OAuth2 'https://www.googleapis.com/auth/userinfo.profile' ]; + /** + * @var array + */ + protected array $iamPermissions = [ + // Database + 'datastore.databases.get', + 'datastore.databases.list', + 'datastore.entities.get', + 'datastore.entities.list', + 'datastore.indexes.get', + 'datastore.indexes.list', + // Generic Firebase permissions + 'firebase.projects.get', + + // Auth + 'firebaseauth.configs.get', + 'firebaseauth.configs.getHashConfig', + 'firebaseauth.configs.getSecret', + 'firebaseauth.users.get', + 'identitytoolkit.tenants.get', + 'identitytoolkit.tenants.list', + + // IAM Assignment + 'iam.serviceAccounts.list', + + // Storage + 'storage.buckets.get', + 'storage.buckets.list', + 'storage.objects.get', + 'storage.objects.list' + ]; + /** * @return string */ @@ -198,7 +230,7 @@ class Firebase extends OAuth2 /* Be careful with the setIAMPolicy method, it will overwrite all existing policies **/ - public function assignIAMRoles(string $accessToken, string $email, string $projectId) + public function assignIAMRole(string $accessToken, string $email, string $projectId, array $role) { // Get IAM Roles $iamRoles = $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':getIamPolicy', [ @@ -209,14 +241,7 @@ class Firebase extends OAuth2 $iamRoles = \json_decode($iamRoles, true); $iamRoles['bindings'][] = [ - 'role' => 'roles/identitytoolkit.admin', - 'members' => [ - 'serviceAccount:' . $email - ] - ]; - - $iamRoles['bindings'][] = [ - 'role' => 'roles/firebase.admin', + 'role' => $role['name'], 'members' => [ 'serviceAccount:' . $email ] @@ -226,7 +251,7 @@ class Firebase extends OAuth2 $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':setIamPolicy', [ 'Authorization: Bearer ' . \urlencode($accessToken), 'Content-Type: application/json' - ], json_encode([ + ], \json_encode([ 'policy' => $iamRoles ])); } @@ -242,6 +267,48 @@ class Firebase extends OAuth2 return $randomString; } + public function createCustomRole(string $accessToken, string $projectId): array + { + // Check if role already exists + try { + $role = $this->request('GET', 'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/appwriteMigrations', [ + 'Content-Type: application/json', + 'Authorization: Bearer ' . \urlencode($accessToken), + ]); + + $role = \json_decode($role, true); + + return $role; + } catch (\Exception $e) { + if ($e->getCode() !== 404) { + throw $e; + } + } + + // Create role if doesn't exist or isn't correct + $role = $this->request( + 'POST', + 'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/', + [ + 'Content-Type: application/json', + 'Authorization: Bearer ' . \urlencode($accessToken), + ], + \json_encode( + [ + 'roleId' => 'appwriteMigrations', + 'role' => [ + 'title' => 'Appwrite Migrations', + 'description' => 'A helper role for Appwrite Migrations', + 'includedPermissions' => $this->iamPermissions, + 'stage' => 'GA' + ] + ] + ) + ); + + return json_decode($role, true); + } + public function createServiceAccount(string $accessToken, string $projectId): array { // Create Service Account @@ -254,7 +321,7 @@ class Firebase extends OAuth2 'Authorization: Bearer ' . \urlencode($accessToken), 'Content-Type: application/json' ], - json_encode([ + \json_encode([ 'accountId' => 'appwrite-' . $uid, 'serviceAccount' => [ 'displayName' => 'Appwrite Migrations ' . $uid @@ -264,7 +331,12 @@ class Firebase extends OAuth2 $response = json_decode($response, true); - $this->assignIAMRoles($accessToken, $response['email'], $projectId); + // Create and assign IAM Roles + $role = $this->createCustomRole($accessToken, $projectId); + + \sleep(1); // Wait for IAM to propagate changes. + + $this->assignIAMRole($accessToken, $response['email'], $projectId, $role); // Create Service Account Key $responseKey = $this->request( @@ -286,10 +358,10 @@ class Firebase extends OAuth2 // List Service Accounts $response = $this->request( 'GET', - 'https://iam.googleapis.com/v1/projects/'.$projectId.'/serviceAccounts', + 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts', [ - 'Authorization: Bearer ' . \urlencode($accessToken), - 'Content-Type: application/json' + 'Authorization: Bearer ' . \urlencode($accessToken), + 'Content-Type: application/json' ] ); From 616e61339c0d934186910aafbdbed7c1d118b84e Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 16 Aug 2023 18:34:22 +0100 Subject: [PATCH 3/9] Update composer.json --- composer.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/composer.json b/composer.json index e2f24388d8..69291bf077 100644 --- a/composer.json +++ b/composer.json @@ -82,10 +82,6 @@ { "url": "https://github.com/appwrite/runtimes.git", "type": "git" - }, - { - "url": "https://github.com/utopia-php/vcs.git", - "type": "git" } ], "require-dev": { From 8222e28ac999a72d24046f56d6fb7b3be1a33fb4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 16 Aug 2023 18:40:24 +0100 Subject: [PATCH 4/9] Update FunctionsCustomServerTest.php --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index df033e63c9..928bba969e 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -733,7 +733,7 @@ class FunctionsCustomServerTest extends Scope ], $this->getHeaders()), [ // Testing default value, should be 'async' => false ]); - + $this->assertEquals(201, $execution['headers']['status-code']); $this->assertEquals('completed', $execution['body']['status']); $this->assertEquals(200, $execution['body']['responseStatusCode']); From 7fedac93068ae2983e4fddff4dd16f75b0e0f543 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 16 Aug 2023 19:08:24 +0100 Subject: [PATCH 5/9] Handle Eldad Suggestions --- app/controllers/general.php | 4 ++-- app/controllers/web/console.php | 2 +- app/init.php | 2 +- src/Appwrite/Auth/OAuth2/Firebase.php | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 7a0cbda9f0..9dbb40c9a8 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -191,7 +191,7 @@ App::init() $host = $request->getHostname() ?? ''; $mainDomain = App::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) { + if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) { if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) { return; } @@ -489,7 +489,7 @@ App::options() $host = $request->getHostname() ?? ''; $mainDomain = App::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) { + if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) { if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) { return; } diff --git a/app/controllers/web/console.php b/app/controllers/web/console.php index 97e9a3932b..d7496b03bd 100644 --- a/app/controllers/web/console.php +++ b/app/controllers/web/console.php @@ -34,7 +34,7 @@ App::get('/console/*') // Serve static files (console) only for main domain $host = $request->getHostname() ?? ''; $mainDomain = App::getEnv('_APP_DOMAIN', ''); - if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) { + if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) { throw new Exception(Exception::GENERAL_ROUTE_NOT_FOUND); } diff --git a/app/init.php b/app/init.php index 79390b8290..0ba8a08d2a 100644 --- a/app/init.php +++ b/app/init.php @@ -135,7 +135,7 @@ const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244'; const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; -const APP_INTERNAL_LOOPBACK_HOST = 'appwrite'; +const APP_HOSTNAME_INTERNAL = 'appwrite'; // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; diff --git a/src/Appwrite/Auth/OAuth2/Firebase.php b/src/Appwrite/Auth/OAuth2/Firebase.php index 93078e5f1d..9c64038089 100644 --- a/src/Appwrite/Auth/OAuth2/Firebase.php +++ b/src/Appwrite/Auth/OAuth2/Firebase.php @@ -256,7 +256,7 @@ class Firebase extends OAuth2 ])); } - function generateRandomString($length = 10) + private function generateRandomString($length = 10): string { $characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; $charactersLength = strlen($characters); @@ -267,7 +267,7 @@ class Firebase extends OAuth2 return $randomString; } - public function createCustomRole(string $accessToken, string $projectId): array + private function createCustomRole(string $accessToken, string $projectId): array { // Check if role already exists try { From 6577fc17a92d77fb41af3ef134692bc3bba66843 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 17 Aug 2023 15:54:19 +0100 Subject: [PATCH 6/9] Remove ./vendor import and add missing envvars --- app/controllers/api/migrations.php | 18 +++++++++++------- docker-compose.yml | 4 +++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 870c5d7d80..33c645ef75 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -453,8 +453,8 @@ App::get('/v1/migrations/appwrite/report') $response ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($appwrite->report($resources)), Response::MODEL_MIGRATION_REPORT); - } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + } catch (\Throwable $e) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); } }); @@ -480,7 +480,7 @@ App::get('/v1/migrations/firebase/report') ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($firebase->report($resources)), Response::MODEL_MIGRATION_REPORT); } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); } }); @@ -564,7 +564,11 @@ App::get('/v1/migrations/firebase/report/oauth') $firebase = new Firebase($serviceAccount); - $report = $firebase->report($resources); + try { + $report = $firebase->report($resources); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); + } $response ->setStatusCode(Response::STATUS_CODE_OK) @@ -869,7 +873,7 @@ App::get('/v1/migrations/supabase/report') ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($supabase->report($resources)), Response::MODEL_MIGRATION_REPORT); } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); } }); @@ -901,7 +905,7 @@ App::get('/v1/migrations/nhost/report') ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($nhost->report($resources)), Response::MODEL_MIGRATION_REPORT); } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); } }); @@ -976,7 +980,7 @@ App::delete('/v1/migrations/:migrationId') } if (!$dbForProject->deleteDocument('migrations', $migration->getId())) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove migration from DB', 500); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove migration from DB'); } $events->setParam('migrationId', $migration->getId()); diff --git a/docker-compose.yml b/docker-compose.yml index 567fb9b96e..64ecc0ae99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -620,11 +620,13 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests - - ./vendor:/usr/src/code/tests depends_on: - mariadb environment: - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET From e8056834aac4faacc121c1f4eace13daba410f97 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 17 Aug 2023 15:55:28 +0100 Subject: [PATCH 7/9] Run Formatter --- app/controllers/api/migrations.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 33c645ef75..a74b460e91 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -454,7 +454,7 @@ App::get('/v1/migrations/appwrite/report') ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($appwrite->report($resources)), Response::MODEL_MIGRATION_REPORT); } catch (\Throwable $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage()); } }); @@ -480,7 +480,7 @@ App::get('/v1/migrations/firebase/report') ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($firebase->report($resources)), Response::MODEL_MIGRATION_REPORT); } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage()); } }); @@ -567,7 +567,7 @@ App::get('/v1/migrations/firebase/report/oauth') try { $report = $firebase->report($resources); } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage()); } $response @@ -873,7 +873,7 @@ App::get('/v1/migrations/supabase/report') ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($supabase->report($resources)), Response::MODEL_MIGRATION_REPORT); } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage()); } }); @@ -905,7 +905,7 @@ App::get('/v1/migrations/nhost/report') ->setStatusCode(Response::STATUS_CODE_OK) ->dynamic(new Document($nhost->report($resources)), Response::MODEL_MIGRATION_REPORT); } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: '.$e->getMessage()); + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage()); } }); From bcfb0d14281d4a74ae6d56fcab49f55ea616ed2f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 18 Aug 2023 11:11:24 +0100 Subject: [PATCH 8/9] Fix Realtime --- src/Appwrite/Messaging/Adapter/Realtime.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index f33f130818..1925944664 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -330,6 +330,11 @@ class Realtime extends Adapter $roles = [Role::team($project->getAttribute('teamId'))->toString()]; } + break; + case 'migrations': + $channels[] = 'console'; + $projectId = 'console'; + $roles = [Role::team($project->getAttribute('teamId'))->toString()]; break; } From 5c40f985f7e3a2656572054c051b9a0ce2221385 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 18 Aug 2023 11:19:51 +0100 Subject: [PATCH 9/9] Run Linter --- app/workers/builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/builds.php b/app/workers/builds.php index f9fc66f2f2..be303b449f 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -335,7 +335,7 @@ class BuildsV1 extends Worker Query::equal('resourceType', ['project']), Query::limit(APP_LIMIT_SUBQUERY) ]); - + foreach ($varsFromProject as $var) { $vars[$var->getAttribute('key')] = $var->getAttribute('value') ?? ''; }