diff --git a/Dockerfile b/Dockerfile index ba8e0ce425..29d495ec26 100755 --- a/Dockerfile +++ b/Dockerfile @@ -244,7 +244,7 @@ RUN \ RUN \ mkdir -p $DOCKER_CONFIG/cli-plugins \ - && ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then $ARCH="armv7"; fi \ + && ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then ARCH="armv7"; fi \ && curl -SL https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-linux-$ARCH -o $DOCKER_CONFIG/cli-plugins/docker-compose \ && chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose diff --git a/app/http.php b/app/http.php index d900947dc7..72aa8183ef 100644 --- a/app/http.php +++ b/app/http.php @@ -118,6 +118,13 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { if (!$dbForConsole->getCollection($key)->isEmpty()) { continue; } + /** + * Skip to prevent 0.15 migration issues. + */ + if ($key === 'databases' && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'collections')) { + continue; + } + Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); $attributes = []; diff --git a/app/realtime.php b/app/realtime.php index 38908e1d2b..d13e1c867f 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -112,6 +112,7 @@ function getDatabase(Registry &$register, string $namespace) if (!$database->exists($database->getDefaultDatabase(), 'realtime')) { throw new Exception('Collection not ready'); } + break; // leave loop if successful } catch (\Throwable $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index ff0705eb32..ff5ec3593f 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -28,9 +28,9 @@ $cli Console::success('Starting Data Migration to version ' . $version); $db = $register->get('db', true); - $cache = $register->get('cache', true); - - $cache = new Cache(new RedisCache($cache)); + $redis = $register->get('cache', true); + $redis->flushAll(); + $cache = new Cache(new RedisCache($redis)); $projectDB = new Database(new MariaDB($db), $cache); $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); @@ -79,5 +79,6 @@ $cli } Swoole\Event::wait(); // Wait for Coroutines to finish + $redis->flushAll(); Console::success('Data Migration Completed'); }); diff --git a/composer.lock b/composer.lock index 4e93839378..0307ffae53 100644 --- a/composer.lock +++ b/composer.lock @@ -2051,16 +2051,16 @@ }, { "name": "utopia-php/database", - "version": "0.18.4", + "version": "0.18.5", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "20aa3893f2f0a970226e54c1cf7d492c44681faa" + "reference": "afdabd8fea127ec39b6e5518f7594b8f2e6c07db" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/20aa3893f2f0a970226e54c1cf7d492c44681faa", - "reference": "20aa3893f2f0a970226e54c1cf7d492c44681faa", + "url": "https://api.github.com/repos/utopia-php/database/zipball/afdabd8fea127ec39b6e5518f7594b8f2e6c07db", + "reference": "afdabd8fea127ec39b6e5518f7594b8f2e6c07db", "shasum": "" }, "require": { @@ -2109,9 +2109,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.18.4" + "source": "https://github.com/utopia-php/database/tree/0.18.5" }, - "time": "2022-06-22T09:18:07+00:00" + "time": "2022-06-24T16:14:12+00:00" }, { "name": "utopia-php/domains", diff --git a/public/dist/scripts/app-all.js b/public/dist/scripts/app-all.js index dd51e3b8f8..57daa23d90 100644 --- a/public/dist/scripts/app-all.js +++ b/public/dist/scripts/app-all.js @@ -3803,7 +3803,7 @@ return false;};return{isRTL:isRTL,};},true);})(window);(function(window){"use st let size=element.dataset["size"]||80;let name=$value.name||$value||"";name=(typeof name!=='string')?'--':name;return def="/v1/avatars/initials?project=console"+"&name="+ encodeURIComponent(name)+"&width="+ size+"&height="+ -size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return date.format("Y-m-d",$value);}).add("dateTime",function($value,date){return date.format("Y-m-d H:i",$value);}).add("dateText",function($value,date){return date.format("d M Y",$value);}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} +size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return $value?date.format("Y-m-d",$value):"";}).add("dateTime",function($value,date){return $value?date.format("Y-m-d H:i",$value):"";}).add("dateText",function($value,date){return $value?date.format("d M Y",$value):"";}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} let value=seconds;if(seconds>=31536000){value=Math.floor(seconds/31536000);unit="year";} else if(seconds>=86400){value=Math.floor(seconds/86400);unit="day";} else if(seconds>=3600){value=Math.floor(seconds/3600);unit="hour";} diff --git a/public/dist/scripts/app.js b/public/dist/scripts/app.js index 35020604e6..639910fc8d 100644 --- a/public/dist/scripts/app.js +++ b/public/dist/scripts/app.js @@ -655,7 +655,7 @@ return false;};return{isRTL:isRTL,};},true);})(window);(function(window){"use st let size=element.dataset["size"]||80;let name=$value.name||$value||"";name=(typeof name!=='string')?'--':name;return def="/v1/avatars/initials?project=console"+"&name="+ encodeURIComponent(name)+"&width="+ size+"&height="+ -size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return date.format("Y-m-d",$value);}).add("dateTime",function($value,date){return date.format("Y-m-d H:i",$value);}).add("dateText",function($value,date){return date.format("d M Y",$value);}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} +size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return $value?date.format("Y-m-d",$value):"";}).add("dateTime",function($value,date){return $value?date.format("Y-m-d H:i",$value):"";}).add("dateText",function($value,date){return $value?date.format("d M Y",$value):"";}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} let value=seconds;if(seconds>=31536000){value=Math.floor(seconds/31536000);unit="year";} else if(seconds>=86400){value=Math.floor(seconds/86400);unit="day";} else if(seconds>=3600){value=Math.floor(seconds/3600);unit="hour";} diff --git a/public/scripts/filters.js b/public/scripts/filters.js index bc8adfc279..eacac40c34 100644 --- a/public/scripts/filters.js +++ b/public/scripts/filters.js @@ -29,13 +29,13 @@ window.ls.filter return !Number.isNaN($value) ? $value.toLocaleString() : ""; }) .add("date", function ($value, date) { - return date.format("Y-m-d", $value); + return $value ? date.format("Y-m-d", $value) : ""; }) .add("dateTime", function ($value, date) { - return date.format("Y-m-d H:i", $value); + return $value ? date.format("Y-m-d H:i", $value) : ""; }) .add("dateText", function ($value, date) { - return date.format("d M Y", $value); + return $value ? date.format("d M Y", $value) : ""; }) .add("timeSince", function ($value) { $value = $value * 1000; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 92c5378425..a133af08ac 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -45,7 +45,7 @@ abstract class Migration '0.14.0' => 'V13', '0.14.1' => 'V13', '0.14.2' => 'V13', - '0.15.0' => 'V13' + '0.15.0' => 'V14' ]; /** @@ -104,8 +104,9 @@ abstract class Migration foreach ($this->collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { - return; + continue; } + $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collection['$id']); @@ -128,7 +129,7 @@ abstract class Migration $old = $document->getArrayCopy(); $new = call_user_func($callback, $document); - if (!self::hasDifference($new->getArrayCopy(), $old)) { + if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) { return; } @@ -228,6 +229,87 @@ abstract class Migration } } + /** + * Creates attribute from collections.php + * + * @param \Utopia\Database\Database $database + * @param string $collectionId + * @param string $attributeId + * @return void + * @throws \Exception + * @throws \Utopia\Database\Exception\Duplicate + * @throws \Utopia\Database\Exception\Limit + */ + public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void + { + $from ??= $collectionId; + $collection = Config::getParam('collections', [])[$from] ?? null; + if (is_null($collection)) { + throw new Exception("Collection {$collectionId} not found"); + } + $attributes = $collection['attributes']; + + $attributeKey = array_search($attributeId, array_column($attributes, '$id')); + + if ($attributeKey === false) { + throw new Exception("Attribute {$attributeId} not found"); + } + + $attribute = $attributes[$attributeKey]; + + $database->createAttribute( + collection: $collectionId, + id: $attributeId, + type: $attribute['type'], + size: $attribute['size'], + required: $attribute['required'] ?? false, + default: $attribute['default'] ?? null, + signed: $attribute['signed'] ?? false, + array: $attribute['array'] ?? false, + format: $attribute['format'] ?? '', + formatOptions: $attribute['formatOptions'] ?? [], + filters: $attribute['filters'] ?? [], + ); + } + + /** + * Creates index from collections.php + * + * @param \Utopia\Database\Database $database + * @param string $collectionId + * @param string $indexId + * @return void + * @throws \Exception + * @throws \Utopia\Database\Exception\Duplicate + * @throws \Utopia\Database\Exception\Limit + */ + public function createIndexFromCollection(Database $database, string $collectionId, string $indexId): void + { + $collection = Config::getParam('collections', [])[$collectionId] ?? null; + + if (is_null($collection)) { + throw new Exception("Collection {$collectionId} not found"); + } + $indexes = $collection['indexes']; + + $indexKey = array_search($indexId, array_column($indexes, '$id')); + + if ($indexKey === false) { + throw new Exception("Attribute {$indexId} not found"); + } + + $index = $indexes[$indexKey]; + + $database->createIndex( + collection: $collectionId, + id: $indexId, + type: $index['type'], + attributes: $index['attributes'], + lengths: $index['lengths'] ?? [], + orders: $index['orders'] ?? [] + ); + } + /** * Executes migration for set project. */ diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php new file mode 100644 index 0000000000..308850d2c9 --- /dev/null +++ b/src/Appwrite/Migration/Version/V14.php @@ -0,0 +1,770 @@ +pdo = $register->get('db'); + + if ($this->project->getId() === 'console' && $this->project->getInternalId() !== 'console') { + return; + } + + /** + * Disable SubQueries for Speed. + */ + foreach (['subQueryAttributes', 'subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships'] as $name) { + Database::addFilter($name, fn () => null, fn () => []); + } + + Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + Console::info('Migrating Collections'); + $this->migrateCollections(); + Console::info('Create Default Database Layer'); + $this->createDatabaseLayer(); + if ($this->project->getId() !== 'console') { + Console::info('Migrating Database Collections'); + $this->migrateCustomCollections(); + } + Console::info('Migrating Documents'); + $this->forEachDocument([$this, 'fixDocument']); + } + + /** + * Creates the default Database for existing Projects. + * + * @return void + * @throws \Throwable + */ + public function createDatabaseLayer(): void + { + try { + if (!$this->projectDB->exists('databases')) { + $this->createCollection('databases'); + } + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + if ($this->project->getInternalId() === 'console') { + return; + } + + try { + $this->projectDB->createDocument('databases', new Document([ + '$id' => 'default', + 'name' => 'Default', + 'search' => 'default Default' + ])); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + } + + /** + * Migrates all Files. + * + * @param \Utopia\Database\Document $bucket + * @return void + * @throws \Exception + */ + protected function migrateBucketFiles(Document $bucket): void + { + $nextFile = null; + do { + $documents = $this->projectDB->find("bucket_{$bucket->getInternalId()}", limit: $this->limit, cursor: $nextFile); + $count = count($documents); + + foreach ($documents as $document) { + go(function (Document $bucket, Document $document) { + Console::log("Migrating File {$document->getId()}"); + try { + /** + * Migrate $createdAt. + */ + if (empty($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + $this->projectDB->updateDocument("bucket_{$bucket->getInternalId()}", $document->getId(), $document); + } + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + }, $bucket, $document); + } + + if ($count !== $this->limit) { + $nextFile = null; + } else { + $nextFile = end($documents); + } + } while (!is_null($nextFile)); + } + + /** + * Migrates all Database Collections. + * @return void + * @throws \Exception + */ + protected function migrateCustomCollections(): void + { + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections` RENAME TO `_{$this->project->getInternalId()}_database_1`")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections_perms` RENAME TO `_{$this->project->getInternalId()}_database_1_perms`")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + /** + * Update metadata table. + */ + try { + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` + SET + _uid = 'database_1', + name = 'database_1' + WHERE _uid = 'collections'; + ")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + try { + /** + * Add Database ID for Collections. + */ + $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseId', 'collections'); + + /** + * Add Database Internal ID for Collections. + */ + $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseInternalId', 'collections'); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + $nextCollection = null; + + do { + $documents = $this->projectDB->find('database_1', limit: $this->limit, cursor: $nextCollection); + $count = count($documents); + + \Co\run(function (array $documents) { + foreach ($documents as $document) { + go(function (Document $collection) { + $id = $collection->getId(); + $internalId = $collection->getInternalId(); + + Console::log("- {$id} ({$collection->getAttribute('name')})"); + + try { + /** + * Rename user's colletion table schema + */ + $this->createNewMetaData("collection_{$internalId}", "database_1_collection_{$internalId}"); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + try { + /** + * Update metadata table. + */ + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` + SET + _uid = 'database_1_collection_{$internalId}', + name = 'database_1_collection_{$internalId}' + WHERE _uid = 'collection_{$internalId}'; + ")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + try { + /** + * Update internal ID's. + */ + $collection + ->setAttribute('databaseId', 'default') + ->setAttribute('databaseInternalId', '1'); + $this->projectDB->updateDocument('database_1', $collection->getId(), $collection); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + }, $document); + } + }, $documents); + + if ($count !== $this->limit) { + $nextCollection = null; + } else { + $nextCollection = end($documents); + } + } while (!is_null($nextCollection)); + } + + /** + * Migrate all Collections. + * + * @return void + */ + protected function migrateCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + Console::log("- {$id}"); + + $this->createNewMetaData($id); + + $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + + switch ($id) { + case 'attributes': + case 'indexes': + try { + /** + * Create 'databaseInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'databaseId'); + } catch (\Throwable $th) { + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'databaseInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'databaseInternalId'); + } catch (\Throwable $th) { + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create 'collectionInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'collectionInternalId'); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Re-Create '_key_collection' index + */ + @$this->projectDB->deleteIndex($id, '_key_collection'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_db_collection'); + } catch (\Throwable $th) { + Console::warning("'_key_collection' from {$id}: {$th->getMessage()}"); + } + + break; + case 'projects': + try { + /** + * Create 'teamInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + break; + case 'platforms': + case 'domains': + try { + /** + * Create 'projectInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_project' index + */ + @$this->projectDB->deleteIndex($id, '_key_project'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'keys': + try { + /** + * Create 'projectInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'expire' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'expire'); + } catch (\Throwable $th) { + Console::warning("'expire' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_project' index + */ + @$this->projectDB->deleteIndex($id, '_key_project'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'webhooks': + try { + /** + * Create 'signatureKey' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'signatureKey'); + } catch (\Throwable $th) { + Console::warning("'signatureKey' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'projectInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_project' index + */ + @$this->projectDB->deleteIndex($id, '_key_project'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'users': + try { + /** + * Create 'phone' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'phone'); + } catch (\Throwable $th) { + Console::warning("'phone' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'phoneVerification' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'phoneVerification'); + } catch (\Throwable $th) { + Console::warning("'phoneVerification' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create '_key_phone' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_phone'); + } catch (\Throwable $th) { + Console::warning("'_key_phone' from {$id}: {$th->getMessage()}"); + } + + break; + case 'tokens': + case 'sessions': + try { + /** + * Create 'userInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_user' index + */ + @$this->projectDB->deleteIndex($id, '_key_user'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); + } catch (\Throwable $th) { + Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); + } + + break; + case 'memberships': + try { + /** + * Create 'teamInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'userInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_unique' index + */ + @$this->projectDB->deleteIndex($id, '_key_unique'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_unique'); + } catch (\Throwable $th) { + Console::warning("'_key_unique' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_team' index + */ + @$this->projectDB->deleteIndex($id, '_key_team'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_team'); + } catch (\Throwable $th) { + Console::warning("'_key_team' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_user' index + */ + @$this->projectDB->deleteIndex($id, '_key_user'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); + } catch (\Throwable $th) { + Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); + } + break; + } + usleep(50000); + } + } + + /** + * Fix run on each document + * + * @param \Utopia\Database\Document $document + * @return \Utopia\Database\Document + */ + protected function fixDocument(Document $document) + { + switch ($document->getCollection()) { + case 'projects': + /** + * Bump Project version number. + */ + $document->setAttribute('version', '0.15.0'); + + if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) { + $internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId(); + $document->setAttribute('teamInternalId', $internalId); + } + + break; + case 'keys': + /** + * Add new 'expire' attribute and default to never (0). + */ + if (is_null($document->getAttribute('expire'))) { + $document->setAttribute('expire', 0); + } + /** + * Add Internal ID 'projectId' for Subqueries. + */ + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; + case 'audit': + /** + * Add Database Layer to collection resource. + */ + if (str_starts_with($document->getAttribute('resource'), 'collection/')) { + $document + ->setAttribute('resource', "database/default/{$document->getAttribute('resource')}") + ->setAttribute('event', "databases.default.{$document->getAttribute('event')}"); + } + + if (str_starts_with($document->getAttribute('resource'), 'document/')) { + $collectionId = explode('.', $document->getAttribute('event'))[1]; + $document + ->setAttribute('resource', "database/default/collection/{$collectionId}/{$document->getAttribute('resource')}") + ->setAttribute('event', "databases.default.{$document->getAttribute('event')}"); + } + + break; + case 'stats': + /** + * Add Database Layer to stats metric. + */ + if (str_starts_with($document->getAttribute('metric'), 'database.')) { + $metric = ltrim($document->getAttribute('metric'), 'database.'); + $document->setAttribute('metric', "databases.default.{$metric}"); + } + + break; + case 'webhooks': + /** + * Add new 'signatureKey' attribute and generate a random value. + */ + if (empty($document->getAttribute('signatureKey'))) { + $document->setAttribute('signatureKey', \bin2hex(\random_bytes(64))); + } + /** + * Add Internal ID 'projectId' for Subqueries. + */ + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; + case 'domains': + /** + * Add Internal ID 'projectId' for Subqueries. + */ + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; + case 'tokens': + case 'sessions': + /** + * Add Internal ID 'userId' for Subqueries. + */ + if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { + $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); + $document->setAttribute('userInternalId', $internalId); + } + + break; + case 'memberships': + /** + * Add Internal ID 'userId' for Subqueries. + */ + if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { + $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); + $document->setAttribute('userInternalId', $internalId); + } + /** + * Add Internal ID 'teamId' for Subqueries. + */ + if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) { + $internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId(); + $document->setAttribute('teamInternalId', $internalId); + } + + break; + case 'attributes': + case 'indexes': + /** + * Add Internal ID 'collectionId' for Subqueries. + */ + if (!empty($document->getAttribute('collectionId')) && is_null($document->getAttribute('collectionInternalId'))) { + $internalId = $this->projectDB->getDocument('database_1', $document->getAttribute('collectionId'))->getInternalId(); + $document->setAttribute('collectionInternalId', $internalId); + } + /** + * Add Internal ID 'databaseInternalId' for Subqueries. + */ + if (is_null($document->getAttribute('databaseInternalId'))) { + $document->setAttribute('databaseInternalId', '1'); + } + /** + * Add Internal ID 'databaseInternalId' for Subqueries. + */ + if (is_null($document->getAttribute('databaseId'))) { + $document->setAttribute('databaseId', 'default'); + } + + try { + /** + * Re-create Collection Document + */ + $this->projectDB->deleteDocument($document->getCollection(), $document->getId()); + $this->projectDB->createDocument($document->getCollection(), $document->setAttribute('$id', "1_{$document->getInternalId()}_{$document->getAttribute('key')}")); + } catch (\Throwable $th) { + Console::warning("Create Collection Document - {$th->getMessage()}"); + } + $document = null; + break; + case 'platforms': + /** + * Migrate dateCreated to $createdAt. + */ + if (empty($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + /** + * Migrate dateUpdated to $updatedAt. + */ + if (empty($document->getUpdatedAt())) { + $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + } + /** + * Add Internal ID 'projectId' for Subqueries. + */ + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; + case 'buckets': + /** + * Migrate dateCreated to $createdAt. + */ + if (empty($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + /** + * Migrate dateUpdated to $updatedAt. + */ + if (empty($document->getUpdatedAt())) { + $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + } + + /** + * Migrate all Storage Buckets to use Internal ID. + */ + $internalId = $this->projectDB->getDocument('buckets', $document->getId())->getInternalId(); + $this->createNewMetaData("bucket_{$internalId}"); + + /** + * Migrate all Storage Bucket Files. + */ + $this->migrateBucketFiles($document); + + break; + case 'users': + /** + * Set 'phoneVerification' to false if not set. + */ + if (is_null($document->getAttribute('phoneVerification'))) { + $document->setAttribute('phoneVerification', false); + } + + break; + case 'functions': + /** + * Migrate dateCreated to $createdAt. + */ + if (empty($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + /** + * Migrate dateUpdated to $updatedAt. + */ + if (empty($document->getUpdatedAt())) { + $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + } + + break; + case 'deployments': + case 'executions': + case 'teams': + /** + * Migrate dateCreated to $createdAt. + */ + if (empty($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + + break; + } + + return $document; + } + + /** + * Creates new metadata that was introduced for a collection and enforces the Internal ID. + * + * @param string $id + * @return void + */ + protected function createNewMetaData(string $id, string $to = null): void + { + $to ??= $id; + /** + * Skip files collection. + */ + if (in_array($id, ['files', 'databases'])) { + return; + } + + try { + /** + * Replace project UID with Internal ID. + */ + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$to}`")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Replace project UID with Internal ID on permissions table. + */ + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$to}_perms`")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Add _createdAt attribute. + */ + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Add _updatedAt attribute. + */ + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Create index for _createdAt. + */ + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$to}` (`_createdAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Create index for _updatedAt. + */ + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$to}` (`_updatedAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + } +} diff --git a/tests/unit/Migration/MigrationTest.php b/tests/unit/Migration/MigrationTest.php index 0c8c34cbc8..24aab8d1e3 100644 --- a/tests/unit/Migration/MigrationTest.php +++ b/tests/unit/Migration/MigrationTest.php @@ -4,7 +4,6 @@ namespace Appwrite\Tests; use Appwrite\Migration\Migration; use PHPUnit\Framework\TestCase; -use ReflectionClass; use ReflectionMethod; use Utopia\Database\Document; diff --git a/tests/unit/Migration/MigrationV13Test.php b/tests/unit/Migration/MigrationV13Test.php index 30e3038564..ff04403536 100644 --- a/tests/unit/Migration/MigrationV13Test.php +++ b/tests/unit/Migration/MigrationV13Test.php @@ -38,15 +38,4 @@ class MigrationV13Test extends MigrationTest $this->assertEquals($document->getAttribute('events'), ['users.*.create']); } - - // This fails due to event validator update - // public function testEventsConversion() - // { - // $migration = new V13(); - // $events = $migration->migrateEvents($migration->events); - // foreach ($events as $event) { - // $this->assertTrue((new Event())->isValid($event), $event); - // } - // $this->assertCount(44, $events); - // } } diff --git a/tests/unit/Migration/MigrationV14Test.php b/tests/unit/Migration/MigrationV14Test.php new file mode 100644 index 0000000000..ea839465af --- /dev/null +++ b/tests/unit/Migration/MigrationV14Test.php @@ -0,0 +1,172 @@ +migration = new V14(); + $reflector = new ReflectionClass('Appwrite\Migration\Version\V14'); + $this->method = $reflector->getMethod('fixDocument'); + $this->method->setAccessible(true); + } + + public function testMigrateProjects() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'projects', + 'version' => '0.14.0' + ])); + + $this->assertEquals($document->getAttribute('version'), '0.15.0'); + $this->assertEquals($document->getAttribute('version'), '0.15.0'); + } + + public function testMigrateKeys() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'keys' + ])); + + $this->assertArrayHasKey('expire', $document->getArrayCopy()); + $this->assertEquals($document->getAttribute('expire'), 0); + } + + public function testMigrateWebhooks() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'webhooks' + ])); + + $this->assertArrayHasKey('signatureKey', $document->getArrayCopy()); + $this->assertEquals(strlen($document->getAttribute('signatureKey')), 128); + } + + public function testMigrateUsers() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'users', + 'phoneVerification' => null + ])); + + $this->assertArrayHasKey('phoneVerification', $document->getArrayCopy()); + $this->assertFalse($document->getAttribute('phoneVerification')); + } + + public function testMigratePlatforms() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'platforms', + '$createdAt' => null, + '$updatedAt' => null, + 'dateCreated' => 123456789, + 'dateUpdated' => 987654321 + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + $this->assertEquals($document->getUpdatedAt(), 987654321); + } + + public function testMigrateFunctions() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'functions', + '$createdAt' => null, + '$updatedAt' => null, + 'dateCreated' => 123456789, + 'dateUpdated' => 987654321 + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + $this->assertEquals($document->getUpdatedAt(), 987654321); + } + + public function testMigrateDeployments() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'deployments', + '$createdAt' => null, + 'dateCreated' => 123456789, + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + } + + public function testMigrateExecutions() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'executions', + '$createdAt' => null, + 'dateCreated' => 123456789, + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + } + + public function testMigrateTeams() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'teams', + '$createdAt' => null, + 'dateCreated' => 123456789, + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + } + + public function testMigrateAudits() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'audit', + 'resource' => 'collection/movies', + 'event' => 'collections.movies.create' + ])); + + $this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies'); + $this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.create'); + + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'audit', + 'resource' => 'document/avatar', + 'event' => 'collections.movies.documents.avatar.create' + ])); + + $this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies/document/avatar'); + $this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.documents.avatar.create'); + } + + public function testMigrateStats() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'stats', + 'metric' => 'database.collections.62b2039844d4277495d0.documents.create' + ])); + + $this->assertEquals($document->getAttribute('metric'), 'databases.default.collections.62b2039844d4277495d0.documents.create'); + + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'stats', + 'metric' => 'users.create' + ])); + + $this->assertEquals($document->getAttribute('metric'), 'users.create'); + } +}