From 5d6b74e447a71b2d23ee594baac1a60194616685 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 14:52:17 +0900 Subject: [PATCH 01/12] Add migrations and filters for 1.6.x --- app/controllers/general.php | 9 +- src/Appwrite/Migration/Migration.php | 1 + src/Appwrite/Migration/Version/V21.php | 143 ++++++++++++++++++ src/Appwrite/Utopia/Request/Filters/V18.php | 21 +++ src/Appwrite/Utopia/Response/Filters/V18.php | 36 +++++ tests/unit/Utopia/Request/Filters/V18Test.php | 51 +++++++ .../unit/Utopia/Response/Filters/V18Test.php | 84 ++++++++++ 7 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 src/Appwrite/Migration/Version/V21.php create mode 100644 src/Appwrite/Utopia/Request/Filters/V18.php create mode 100644 src/Appwrite/Utopia/Response/Filters/V18.php create mode 100644 tests/unit/Utopia/Request/Filters/V18Test.php create mode 100644 tests/unit/Utopia/Response/Filters/V18Test.php diff --git a/app/controllers/general.php b/app/controllers/general.php index 10c9eb8e18..72143cbeca 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -11,9 +11,10 @@ use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Request; use Appwrite\Utopia\Request\Filters\V16 as RequestV16; use Appwrite\Utopia\Request\Filters\V17 as RequestV17; +use Appwrite\Utopia\Request\Filters\V18 as RequestV18; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Filters\V16 as ResponseV16; -use Appwrite\Utopia\Response\Filters\V17 as ResponseV17; +use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; @@ -434,6 +435,9 @@ App::init() if (version_compare($requestFormat, '1.5.0', '<')) { $request->addFilter(new RequestV17()); } + if (version_compare($requestFormat, '1.6.0', '<')) { + $request->addFilter(new RequestV18()); + } } $domain = $request->getHostname(); @@ -550,6 +554,9 @@ App::init() if (version_compare($responseFormat, '1.5.0', '<')) { $response->addFilter(new ResponseV17()); } + if (version_compare($responseFormat, '1.6.0', '<')) { + $response->addFilter(new ResponseV18()); + } if (version_compare($responseFormat, APP_VERSION_STABLE, '>')) { $response->addHeader('X-Appwrite-Warning', "The current SDK is built for Appwrite " . $responseFormat . ". However, the current Appwrite server version is ". APP_VERSION_STABLE . ". Please downgrade your SDK to match the Appwrite version: https://appwrite.io/docs/sdks"); } diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index e3a2021c1a..716f6c6381 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -86,6 +86,7 @@ abstract class Migration '1.5.5' => 'V20', '1.5.6' => 'V20', '1.5.7' => 'V20', + '1.6.0' => 'V12' ]; /** diff --git a/src/Appwrite/Migration/Version/V21.php b/src/Appwrite/Migration/Version/V21.php new file mode 100644 index 0000000000..7dc9e4374b --- /dev/null +++ b/src/Appwrite/Migration/Version/V21.php @@ -0,0 +1,143 @@ + null, + fn () => [] + ); + } + + Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + + Console::info('Migrating Collections'); + $this->migrateCollections(); + + Console::info('Migrating Documents'); + $this->forEachDocument([$this, 'fixDocument']); + } + + /** + * Migrate Collections. + * + * @return void + * @throws Exception|Throwable + */ + private function migrateCollections(): void + { + $internalProjectId = $this->project->getInternalId(); + $collectionType = match ($internalProjectId) { + 'console' => 'console', + default => 'projects', + }; + + $collections = $this->collections[$collectionType]; + foreach ($collections as $collection) { + $id = $collection['$id']; + + Console::log("Migrating Collection \"{$id}\""); + + $this->projectDB->setNamespace("_$internalProjectId"); + + switch ($id) { + case 'projects': + // Create accessedAt attribute + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt'); + } catch (Throwable $th) { + Console::warning("'accessedAt' from {$id}: {$th->getMessage()}"); + } + break; + case 'schedules': + // Create data attribute + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'data'); + } catch (Throwable $th) { + Console::warning("'data' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'functions': + // Create scopes attribute + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'scopes'); + } catch (Throwable $th) { + Console::warning("'scopes' from {$id}: {$th->getMessage()}"); + } + + break; + case 'executions': + // Create requestMethod index + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_requestMethod'); + } catch (\Throwable $th) { + Console::warning("'_key_requestMethod' from {$id}: {$th->getMessage()}"); + } + + // Create requestPath index + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_requestPath'); + } catch (\Throwable $th) { + Console::warning("'_key_requestPath' from {$id}: {$th->getMessage()}"); + } + + // Create deployment index + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_deployment'); + } catch (\Throwable $th) { + Console::warning("'_key_deployment' from {$id}: {$th->getMessage()}"); + } + } + + usleep(50000); + } + } + + /** + * Fix run on each document + * + * @param Document $document + * @return Document + */ + protected function fixDocument(Document $document): Document + { + switch ($document->getCollection()) { + case 'projects': + /** + * Bump version number. + */ + $document->setAttribute('version', '1.6.0'); + break; + case 'functions': + /** + * Add scopes attribute. + */ + if (!$document->getAttribute('scopes', false)) { + $document->setAttribute('scopes', []); + } + break; + } + return $document; + } +} diff --git a/src/Appwrite/Utopia/Request/Filters/V18.php b/src/Appwrite/Utopia/Request/Filters/V18.php new file mode 100644 index 0000000000..fd8b7ffd3a --- /dev/null +++ b/src/Appwrite/Utopia/Request/Filters/V18.php @@ -0,0 +1,21 @@ + $this->parseFunction($content), + Response::MODEL_PROJECT => $this->parseProject($content), + default => $parsedResponse, + }; + + return $parsedResponse; + } + + protected function parseFunction(array $content) + { + unset($content['scopes']); + return $content; + } + + protected function parseProject(array $content) + { + unset($content['authMockNumbers']); + unset($content['authSessionAlerts']); + return $content; + } +} \ No newline at end of file diff --git a/tests/unit/Utopia/Request/Filters/V18Test.php b/tests/unit/Utopia/Request/Filters/V18Test.php new file mode 100644 index 0000000000..4e1f81573a --- /dev/null +++ b/tests/unit/Utopia/Request/Filters/V18Test.php @@ -0,0 +1,51 @@ +filter = new V18(); + } + + public function tearDown(): void + { + } + + public function deleteMfaAuthenticatorProvider() + { + return [ + 'remove otp' => [ + [ + 'type' => 'totp', + 'otp' => 1230 + ], + [ + 'type' => 'totp' + ] + ] + ]; + } + + /** + * @dataProvider deleteMfaAuthenticatorProvider + */ + public function testdeleteMfaAuthenticator(array $content, array $expected): void + { + $model = 'account.deleteMfaAuthenticator'; + + $result = $this->filter->parse($content, $model); + + $this->assertEquals($expected, $result); + } +} diff --git a/tests/unit/Utopia/Response/Filters/V18Test.php b/tests/unit/Utopia/Response/Filters/V18Test.php new file mode 100644 index 0000000000..0b12ca66cd --- /dev/null +++ b/tests/unit/Utopia/Response/Filters/V18Test.php @@ -0,0 +1,84 @@ +filter = new V18(); + } + + public function tearDown(): void + { + } + + public function functionProvider(): array + { + return [ + 'remove scopes' => [ + [ + 'scopes' => [ + 'example_scope', + 'example_scope2', + ], + ], + [ + ] + ] + ]; + } + + /** + * @dataProvider functionProvider + */ + public function testFunction(array $content, array $expected): void + { + $model = Response::MODEL_FUNCTION; + + $result = $this->filter->parse($content, $model); + + $this->assertEquals($expected, $result); + } + + public function projectProvider(): array + { + return [ + 'remove authMockNumbers and authSessionAlerts' => [ + [ + 'authMockNumbers' => [ + 'example_mock_number', + 'example_mock_number2', + ], + 'authSessionAlerts' => [ + 'example_alert', + 'example_alert2', + ], + ], + [ + ] + ] + ]; + } + + /** + * @dataProvider projectProvider + */ + public function testProject(array $content, array $expected): void + { + $model = Response::MODEL_PROJECT; + + $result = $this->filter->parse($content, $model); + + $this->assertEquals($expected, $result); + } +} \ No newline at end of file From ec9bb8a898b7e7a08d1106c33922d6653aa574dd Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 14:53:23 +0900 Subject: [PATCH 02/12] Re-add V17 Response filter --- app/controllers/general.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 72143cbeca..70acd9d482 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -14,6 +14,7 @@ use Appwrite\Utopia\Request\Filters\V17 as RequestV17; use Appwrite\Utopia\Request\Filters\V18 as RequestV18; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Filters\V16 as ResponseV16; +use Appwrite\Utopia\Response\Filters\V17 as ResponseV17; use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; From 35375c42ad38a7266f4a45f1f3383c2ee0236197 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 14:54:37 +0900 Subject: [PATCH 03/12] Fix typos --- src/Appwrite/Migration/Migration.php | 2 +- src/Appwrite/Migration/Version/V21.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 716f6c6381..9075b85702 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -86,7 +86,7 @@ abstract class Migration '1.5.5' => 'V20', '1.5.6' => 'V20', '1.5.7' => 'V20', - '1.6.0' => 'V12' + '1.6.0' => 'V21' ]; /** diff --git a/src/Appwrite/Migration/Version/V21.php b/src/Appwrite/Migration/Version/V21.php index 7dc9e4374b..10193ec60c 100644 --- a/src/Appwrite/Migration/Version/V21.php +++ b/src/Appwrite/Migration/Version/V21.php @@ -9,7 +9,7 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; -class V20 extends Migration +class V21 extends Migration { /** * @throws Throwable From 917de006bff0fc694140ca04fd293470a936099b Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 15:01:44 +0900 Subject: [PATCH 04/12] Increate stable version number --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index a0e71f041b..ef7742956f 100644 --- a/app/init.php +++ b/app/init.php @@ -118,7 +118,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4314; -const APP_VERSION_STABLE = '1.5.7'; +const APP_VERSION_STABLE = '1.6.0'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; From d78c880bf02aadb8ede51bb3ca50bf59f3b7ff69 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 15:20:34 +0900 Subject: [PATCH 05/12] Update V21.php --- src/Appwrite/Migration/Version/V21.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V21.php b/src/Appwrite/Migration/Version/V21.php index 10193ec60c..4bd27b71bc 100644 --- a/src/Appwrite/Migration/Version/V21.php +++ b/src/Appwrite/Migration/Version/V21.php @@ -7,6 +7,7 @@ use Exception; use Throwable; use Utopia\CLI\Console; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; class V21 extends Migration @@ -133,7 +134,7 @@ class V21 extends Migration /** * Add scopes attribute. */ - if (!$document->getAttribute('scopes', false)) { + if (empty($document->getAttribute('scopes', []))) { $document->setAttribute('scopes', []); } break; From 7dec5f68bef598b1a178de97d051b8e43bb447f7 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 15:20:42 +0900 Subject: [PATCH 06/12] Update V21.php --- src/Appwrite/Migration/Version/V21.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Migration/Version/V21.php b/src/Appwrite/Migration/Version/V21.php index 4bd27b71bc..53d8101af2 100644 --- a/src/Appwrite/Migration/Version/V21.php +++ b/src/Appwrite/Migration/Version/V21.php @@ -129,6 +129,7 @@ class V21 extends Migration * Bump version number. */ $document->setAttribute('version', '1.6.0'); + $document->setAttribute('accessedAt', DateTime::now()); break; case 'functions': /** From 83c43503957171fa253d5d0d73bd540d18a31b09 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 17:58:25 +0900 Subject: [PATCH 07/12] Make scopes optional on functions --- app/config/collections.php | 4 ++-- src/Appwrite/Migration/Version/V21.php | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index d0c3df165e..b4dc1049fb 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4751,8 +4751,8 @@ $consoleCollections = array_merge([ 'format' => '', 'size' => Database::LENGTH_KEY, 'signed' => true, - 'required' => true, - 'default' => null, + 'required' => false, + 'default' => [], 'array' => true, 'filters' => [], ], diff --git a/src/Appwrite/Migration/Version/V21.php b/src/Appwrite/Migration/Version/V21.php index 53d8101af2..c5f09822cb 100644 --- a/src/Appwrite/Migration/Version/V21.php +++ b/src/Appwrite/Migration/Version/V21.php @@ -131,14 +131,6 @@ class V21 extends Migration $document->setAttribute('version', '1.6.0'); $document->setAttribute('accessedAt', DateTime::now()); break; - case 'functions': - /** - * Add scopes attribute. - */ - if (empty($document->getAttribute('scopes', []))) { - $document->setAttribute('scopes', []); - } - break; } return $document; } From efa1924f26649a13471d5f14ad8e8acc113169f9 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 17 Jul 2024 18:00:41 +0900 Subject: [PATCH 08/12] Run Linter --- src/Appwrite/Utopia/Request/Filters/V18.php | 2 +- src/Appwrite/Utopia/Response/Filters/V18.php | 2 +- tests/unit/Utopia/Response/Filters/V18Test.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Utopia/Request/Filters/V18.php b/src/Appwrite/Utopia/Request/Filters/V18.php index fd8b7ffd3a..4f889ce66b 100644 --- a/src/Appwrite/Utopia/Request/Filters/V18.php +++ b/src/Appwrite/Utopia/Request/Filters/V18.php @@ -18,4 +18,4 @@ class V18 extends Filter return $content; } -} \ No newline at end of file +} diff --git a/src/Appwrite/Utopia/Response/Filters/V18.php b/src/Appwrite/Utopia/Response/Filters/V18.php index ea82f8335e..b32856bdb4 100644 --- a/src/Appwrite/Utopia/Response/Filters/V18.php +++ b/src/Appwrite/Utopia/Response/Filters/V18.php @@ -33,4 +33,4 @@ class V18 extends Filter unset($content['authSessionAlerts']); return $content; } -} \ No newline at end of file +} diff --git a/tests/unit/Utopia/Response/Filters/V18Test.php b/tests/unit/Utopia/Response/Filters/V18Test.php index 0b12ca66cd..36719a7620 100644 --- a/tests/unit/Utopia/Response/Filters/V18Test.php +++ b/tests/unit/Utopia/Response/Filters/V18Test.php @@ -81,4 +81,4 @@ class V18Test extends TestCase $this->assertEquals($expected, $result); } -} \ No newline at end of file +} From 8e61b2149f1b5e538b210dbde947b644904627b4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 22 Jul 2024 11:33:31 +0900 Subject: [PATCH 09/12] Auto update V3 Runtimes into V4 Runtimes on execution --- app/controllers/api/functions.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d2a1790d94..a810d7be5b 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1621,6 +1621,13 @@ App::post('/v1/functions/:functionId/executions') $version = $function->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); + /** Auto-update v3 runtimes into v4, remove me after we do migrations */ + if ($version === 'v3') { + $function->setAttribute('version', 'v4'); + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); + $version = 'v4'; + } + $runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null; if (\is_null($runtime)) { From 9e3c5b9368745e4a06b80d3822612adf68545139 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 22 Jul 2024 19:40:05 +0900 Subject: [PATCH 10/12] Add variable runtimes sizes migrations --- src/Appwrite/Migration/Version/V21.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Appwrite/Migration/Version/V21.php b/src/Appwrite/Migration/Version/V21.php index 01fc2d19e0..be6c38be67 100644 --- a/src/Appwrite/Migration/Version/V21.php +++ b/src/Appwrite/Migration/Version/V21.php @@ -87,6 +87,13 @@ class V21 extends Migration Console::warning("'scopes' from {$id}: {$th->getMessage()}"); } + // Create size attribute + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'size'); + } catch (Throwable $th) { + Console::warning("'size' from {$id}: {$th->getMessage()}"); + } + break; case 'executions': // Create requestMethod index @@ -133,7 +140,14 @@ class V21 extends Migration // Add accessedAt attribute $document->setAttribute('accessedAt', DateTime::now()); break; + case 'functions': + // Add scopes attribute + $document->setAttribute('scopes', []); + + // Add size attribute + $document->setAttribute('size', 's-1vcpu-512m'); } + return $document; } } From 4f558d35be000646725c70f9b3bdda0f70f0829f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 22 Jul 2024 19:42:29 +0900 Subject: [PATCH 11/12] Improvement v3 -> v4 transition --- app/controllers/api/functions.php | 7 ------- src/Executor/Executor.php | 13 +++++++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a810d7be5b..d2a1790d94 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1621,13 +1621,6 @@ App::post('/v1/functions/:functionId/executions') $version = $function->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - /** Auto-update v3 runtimes into v4, remove me after we do migrations */ - if ($version === 'v3') { - $function->setAttribute('version', 'v4'); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - $version = 'v4'; - } - $runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null; if (\is_null($runtime)) { diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 29ed756932..3206381e95 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -75,6 +75,12 @@ class Executor $runtimeId = "$projectId-$deploymentId-build"; $route = "/runtimes"; $timeout = (int) System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900); + + // Remove after migration + if ($version == 'v3') { + $version = 'v4'; + } + $params = [ 'runtimeId' => $runtimeId, 'source' => $source, @@ -188,6 +194,13 @@ class Executor $runtimeId = "$projectId-$deploymentId"; $route = '/runtimes/' . $runtimeId . '/execution'; + + + // Remove after migration + if ($version == 'v3') { + $version = 'v4'; + } + $params = [ 'runtimeId' => $runtimeId, 'variables' => $variables, From 4b585466e1a43930d2efe097c3f7de0e236ce561 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 22 Jul 2024 20:03:45 +0900 Subject: [PATCH 12/12] Update api.php --- app/controllers/shared/api.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 583a0160a1..672366fa01 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -206,6 +206,7 @@ App::init() throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); } + // Remove after migration if(!\str_contains($apiKey, '_')) { $keyType = API_KEY_STANDARD; $authKey = $apiKey;