diff --git a/app/init.php b/app/init.php index 917b1bd7b2..fb2d7ba8c5 100644 --- a/app/init.php +++ b/app/init.php @@ -71,7 +71,7 @@ const APP_LIMIT_ENCRYPTION = 20000000; //20MB const APP_LIMIT_COMPRESSION = 20000000; //20MB const APP_LIMIT_PREVIEW = 10000000; //10MB file size limit for preview endpoint const APP_CACHE_BUSTER = 300; -const APP_VERSION_STABLE = '0.12.3'; +const APP_VERSION_STABLE = '0.13.0'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Version/V12.php b/src/Appwrite/Migration/Version/V12.php index 73177b61fb..3d56f36355 100644 --- a/src/Appwrite/Migration/Version/V12.php +++ b/src/Appwrite/Migration/Version/V12.php @@ -3,9 +3,11 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; class V12 extends Migration { @@ -35,6 +37,7 @@ class V12 extends Migration Console::info('Migrating Permissions'); $this->fixPermissions(); Console::info('Migrating Collections'); + $this->migrateCustomCollections(); $this->fixCollections(); Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); @@ -54,7 +57,7 @@ class V12 extends Migration * Remove empty generated Console Project. */ if ($this->consoleDB->getNamespace() === '_project_console' && $projectId === 'console') { - $all = []; + $all = ['_console_bucket_1', '_console_bucket_1_perms']; foreach ($this->collections as $collection) { $all[] = "_{$projectId}_{$collection['$id']}"; $all[] = "_{$projectId}_{$collection['$id']}_perms"; @@ -70,6 +73,13 @@ class V12 extends Migration foreach ($this->collections as $collection) { $id = $collection['$id']; + /** + * Skip new tables that don't exists on old schema. + */ + if (in_array($id, ['buckets', 'deployments', 'builds'])) { + continue; + } + $this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_{$id} RENAME TO _{$projectId}_{$id}")->execute(); $this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_{$id}_perms ( `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, @@ -92,32 +102,294 @@ class V12 extends Migration { foreach ($this->collections as $collection) { $id = $collection['$id']; + + /** + * Skip new tables that don't exists on old schema. + */ + if (in_array($id, ['buckets', 'deployments', 'builds'])) { + continue; + } Console::log("- {$id}"); switch ($id) { case 'sessions': try { + /** + * Rename providerToken to providerAccessToken + */ $this->projectDB->renameAttribute($id, 'providerToken', 'providerAccessToken'); } catch (\Throwable $th) { Console::warning("'providerAccessToken' from {$id}: {$th->getMessage()}"); } try { + /** + * Create providerRefreshToken + */ $this->projectDB->createAttribute(collection: $id, id: 'providerRefreshToken', type: Database::VAR_STRING, size: 16384, signed: true, required: true, filters: ['encrypt']); } catch (\Throwable $th) { Console::warning("'providerRefreshToken' from {$id}: {$th->getMessage()}"); } try { + /** + * Create providerAccessTokenExpiry + */ $this->projectDB->createAttribute(collection: $id, id: 'providerAccessTokenExpiry', type: Database::VAR_INTEGER, size: 0, required: true); } catch (\Throwable $th) { Console::warning("'providerAccessTokenExpiry' from {$id}: {$th->getMessage()}"); } break; + + case 'memberships': + try { + /** + * Add search attribute and index to memberships. + */ + $this->projectDB->createAttribute(collection: $id, id: 'search', type: Database::VAR_STRING, size: 16384, required: false); + $this->projectDB->createIndex(collection: $id, id: '_key_search', type: Database::INDEX_FULLTEXT, attributes: ['search']); + } catch (\Throwable $th) { + Console::warning("'search' from {$id}: {$th->getMessage()}"); + } + break; + + case 'files': + /** + * Create bucket table if not exists. + */ + $this->createCollection('buckets'); + + if (!$this->projectDB->findOne('buckets', [new Query('$id', Query::TYPE_EQUAL, ['default'])])) { + $this->projectDB->createDocument('buckets', new Document([ + '$id' => 'default', + '$collection' => 'buckets', + 'dateCreated' => \time(), + 'dateUpdated' => \time(), + 'name' => 'Default', + 'permission' => 'file', + 'maximumFileSize' => (int) App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB + 'allowedFileExtensions' => [], + 'enabled' => true, + 'encryption' => true, + 'antivirus' => true, + '$read' => ['role:all'], + '$write' => ['role:all'], + 'search' => 'buckets Default', + ])); + $this->createCollection('files', 'bucket_1'); + + /** + * Rename folder on volumes. + */ + $path = "/storage/uploads/app-{$this->project->getId()}"; + + if (is_dir("{$path}/")) { + mkdir("/storage/uploads/app-{$this->project->getId()}/default"); + + foreach (new \DirectoryIterator($path) as $fileinfo) { + if ($fileinfo->isDir() && !$fileinfo->isDot() && $fileinfo->getFilename() !== 'default') { + rename("{$path}/{$fileinfo->getFilename()}", "{$path}/default/{$fileinfo->getFilename()}"); + } + } + } + } + + break; + + case 'functions': + try { + /** + * Rename tag to deployment + */ + $this->projectDB->renameAttribute($id, 'tag', 'deployment'); + } catch (\Throwable $th) { + Console::warning("'deployment' from {$id}: {$th->getMessage()}"); + } + + /** + * Create deployments table if not exists. + */ + $this->createCollection('deployments'); + + /** + * Create builds table if not exists. + */ + $this->createCollection('builds'); + + break; } usleep(100000); } } + /** + * Creates colletion from the config collection. + * + * @param string $id + * @param string|null $name + * @return void + * @throws \Throwable + */ + protected function createCollection(string $id, string $name = null): void + { + $name ??= $id; + + if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) { + $attributes = []; + $indexes = []; + $collection = $this->collections[$id]; + + foreach ($collection['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); + } + + foreach ($collection['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + try { + $this->projectDB->createCollection($name, $attributes, $indexes); + } catch (\Throwable $th) { + throw $th; + } + } + } + + /** + * Migrates permissions to dedicated table. + * + * @param \Utopia\Database\Document $document + * @param string $internalId + * @return void + * @throws \Exception + * @throws \PDOException + */ + protected function migratePermissionsToDedicatedTable(string $collection, Document $document): void + { + $sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$collection}` WHERE _uid = {$this->pdo->quote($document->getid())}"; + $stmt = $this->pdo->prepare($sql); + $stmt->execute(); + + $permissions = $stmt->fetch(); + + $read = json_decode($permissions['_read'] ?? null) ?? []; + $write = json_decode($permissions['_write'] ?? null) ?? []; + + $permissions = []; + foreach ($read as $permission) { + $permissions[] = "('read', '{$permission}', '{$document->getId()}')"; + } + + foreach ($write as $permission) { + $permissions[] = "('write', '{$permission}', '{$document->getId()}')"; + } + + if (!empty($permissions)) { + $queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$collection}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions); + $stmtPermissions = $this->pdo->prepare($queryPermissions); + $stmtPermissions->execute(); + } + } + + /** + * Migrates all user's database collections. + * + * @return void + * @throws \Exception + */ + protected function migrateCustomCollections(): void + { + $nextCollection = null; + + do { + $documents = $this->projectDB->find('collections', limit: $this->limit, cursor: $nextCollection); + $count = count($documents); + + \Co\run(function (array $documents) { + foreach ($documents as $document) { + go(function (Document $collection) { + $id = $collection->getId(); + $projectId = $this->project->getId(); + $internalId = $collection->getInternalId(); + + /** + * Rename user's colletion table schema + */ + $this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_collection_{$id} RENAME TO _{$projectId}_collection_{$internalId}")->execute(); + $this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_collection_{$internalId}_perms ( + `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `_type` VARCHAR(12) NOT NULL, + `_permission` VARCHAR(255) NOT NULL, + `_document` VARCHAR(255) NOT NULL, + PRIMARY KEY (`_id`), + UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`), + INDEX `_index2` (`_permission`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute(); + + /** + * Update metadata table. + */ + $this->pdo->prepare("UPDATE _{$projectId}__metadata + SET + _uid = 'collection_{$internalId}', + name = 'collection_{$internalId}' + WHERE _uid = 'collection_{$id}'; + ")->execute(); + + + $nextDocument = null; + + do { + $documents = $this->projectDB->find('collection_' . $internalId, limit: $this->limit, cursor: $nextDocument); + $count = count($documents); + + foreach ($documents as $document) { + go(function (Document $document, string $internalId) { + $this->migratePermissionsToDedicatedTable("collection_{$internalId}", $document); + }, $document, $internalId); + } + + if ($count !== $this->limit) { + $nextDocument = null; + } else { + $nextDocument = end($documents); + } + } while (!is_null($nextDocument)); + + /** + * Remove _read and _write columns + */ + $this->pdo->prepare(" + ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_collection_{$internalId}` + DROP COLUMN _read, + DROP COLUMN _write + ")->execute(); + }, $document); + } + }, $documents); + + if ($count !== $this->limit) { + $nextCollection = null; + } else { + $nextCollection = end($documents); + } + } while (!is_null($nextCollection)); + } + /** * Migrate all Permission to new System with dedicated Table. + * * @return void * @throws \Exception */ @@ -125,6 +397,31 @@ class V12 extends Migration { foreach ($this->collections as $collection) { $id = $collection['$id']; + + /** + * Skip new tables that don't exists on old schema. + */ + if (in_array($id, ['buckets', 'deployments', 'builds'])) { + continue; + } + /** + * Check if permissions have already been migrated. + */ + try { + $stmtCheck = $this->pdo->prepare("SHOW COLUMNS from `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$id}` LIKE '_read'"); + $stmtCheck->execute(); + + if (empty($stmtCheck->fetchAll())) { + continue; + } + } catch (\Throwable $th) { + if ($th->getCode() === "42S02") { + continue; + } + throw $th; + } + + Console::log("- {$collection['$id']}"); $nextDocument = null; @@ -135,29 +432,7 @@ class V12 extends Migration \Co\run(function (array $documents) { foreach ($documents as $document) { go(function (Document $document) { - $sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}` WHERE _uid = {$this->pdo->quote($document->getid())}"; - $stmt = $this->pdo->prepare($sql); - $stmt->execute(); - - $permissions = $stmt->fetch(); - - $read = json_decode($permissions['_read'] ?? null) ?? []; - $write = json_decode($permissions['_write'] ?? null) ?? []; - - $permissions = []; - foreach ($read as $permission) { - $permissions[] = "('read', '{$permission}', '{$document->getId()}')"; - } - - foreach ($write as $permission) { - $permissions[] = "('write', '{$permission}', '{$document->getId()}')"; - } - - if (!empty($permissions)) { - $queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions); - $stmtPermissions = $this->pdo->prepare($queryPermissions); - $stmtPermissions->execute(); - } + $this->migratePermissionsToDedicatedTable($document->getCollection(), $document); }, $document); } }, $documents); @@ -168,6 +443,15 @@ class V12 extends Migration $nextDocument = end($documents); } } while (!is_null($nextDocument)); + + /** + * Remove _read and _write columns + */ + $this->pdo->prepare(" + ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$id}` + DROP COLUMN _read, + DROP COLUMN _write + ")->execute(); } /** @@ -176,6 +460,12 @@ class V12 extends Migration usleep(100000); } + /** + * Fix run on each document + * + * @param \Utopia\Database\Document $document + * @return \Utopia\Database\Document + */ protected function fixDocument(Document $document) { switch ($document->getCollection()) { @@ -205,6 +495,13 @@ class V12 extends Migration break; case 'teams': + /** + * Rename sum to total + */ + if (empty($document->getAttribute('total'))) { + $document->setAttribute('total', $document->getAttribute('sum')); + } + /** * Populate search string from Migration to 0.12. */ @@ -215,6 +512,14 @@ class V12 extends Migration break; case 'files': + /** + * Update File Path + */ + $path = "/storage/uploads/app-{$this->project->getId()}"; + $new = str_replace($path, "{$path}/default", $document->getAttribute('path')); + $document + ->setAttribute('bucketId', 'default') + ->setAttribute('path', $new); /** * Populate search string from Migration to 0.12. */ @@ -225,6 +530,8 @@ class V12 extends Migration break; case 'functions': + $document->setAttribute('deployment', null); + /** * Populate search string from Migration to 0.12. */ @@ -234,16 +541,6 @@ class V12 extends Migration break; - case 'tags': - /** - * Populate search string from Migration to 0.12. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'command'], $document)); - } - - break; - case 'executions': /** * Populate search string from Migration to 0.12. @@ -254,6 +551,16 @@ class V12 extends Migration break; + case 'memberships': + /** + * Populate search string. + */ + if (empty($document->getAttribute('search'))) { + $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'userId'], $document)); + } + + break; + case 'sessions': $document ->setAttribute('providerRefreshToken', '')