diff --git a/app/http.php b/app/http.php index da726c306c..afda7053b4 100644 --- a/app/http.php +++ b/app/http.php @@ -38,8 +38,7 @@ $http 'http_compression_level' => 6, 'package_max_length' => $payloadSize, 'buffer_output_size' => $payloadSize, - ]) -; + ]); $http->on('WorkerStart', function ($server, $workerId) { Console::success('Worker ' . ++$workerId . ' started successfully'); @@ -81,13 +80,16 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { } } while ($attempts < $max); - App::setResource('db', fn() => $db); - App::setResource('cache', fn() => $redis); + App::setResource('db', fn () => $db); + App::setResource('cache', fn () => $redis); - $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForConsole */ + $dbForConsole = $app->getResource('dbForConsole'); Console::success('[Setup] - Server database init started...'); - $collections = Config::getParam('collections', []); /** @var array $collections */ + + /** @var array $collections */ + $collections = Config::getParam('collections', []); if (!$dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'))) { $redis->flushAll(); @@ -122,9 +124,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { continue; } /** - * Skip to prevent 0.15 migration issues. + * Skip to prevent 0.16 migration issues. */ - if ($key === 'databases' && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'collections')) { + if (in_array($key, ['cache', 'variables']) && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { continue; } @@ -160,7 +162,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection($key, $attributes, $indexes); } - if ($dbForConsole->getDocument('buckets', 'default')->isEmpty()) { + if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { Console::success('[Setup] - Creating default bucket...'); $dbForConsole->createDocument('buckets', new Document([ '$id' => ID::custom('default'), @@ -244,8 +246,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo ->setContentType(Files::getFileMimeType($request->getURI())) ->addHeader('Cache-Control', 'public, max-age=' . $time) ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time) . ' GMT') // 45 days cache - ->send(Files::getFileContents($request->getURI())) - ; + ->send(Files::getFileContents($request->getURI())); return; } @@ -255,8 +256,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $db = $register->get('dbPool')->get(); $redis = $register->get('redisPool')->get(); - App::setResource('db', fn() => $db); - App::setResource('cache', fn() => $redis); + App::setResource('db', fn () => $db); + App::setResource('cache', fn () => $redis); try { Authorization::cleanRoles(); diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index c950cf1bf6..f0ab71a964 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -15,7 +15,7 @@ use Utopia\Validator\Text; $cli ->task('migrate') - ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) + ->param('version', APP_VERSION_STABLE, new Text(32), 'Version to migrate to.', true) ->action(function ($version) use ($register) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -45,6 +45,9 @@ $cli $limit = 30; $sum = 30; $offset = 0; + /** + * @var \Utopia\Database\Document[] $projects + */ $projects = [$console]; $count = 0; @@ -60,6 +63,13 @@ $cli while (!empty($projects)) { foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console' && $project->getInternalId() !== 'console') { + continue; + } + try { $migration ->setProject($project, $projectDB, $consoleDB) diff --git a/phpunit.xml b/phpunit.xml index 97bec18b42..4074fe0f1c 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -1,6 +1,6 @@ 'V12', - '0.13.1' => 'V12', - '0.13.2' => 'V12', - '0.13.3' => 'V12', - '0.13.4' => 'V12', - '0.14.0' => 'V13', - '0.14.1' => 'V13', - '0.14.2' => 'V13', - '0.15.0' => 'V14', - '0.15.1' => 'V14', - '0.15.2' => 'V14', - '0.15.3' => 'V14' + '1.0.0-RC1' => 'V15', + '1.0.0' => 'V15' ]; /** @@ -62,6 +54,7 @@ abstract class Migration { Authorization::disable(); Authorization::setDefaultStatus(false); + $this->collections = array_merge([ '_metadata' => [ '$id' => ID::custom('_metadata'), @@ -105,66 +98,74 @@ abstract class Migration */ public function forEachDocument(callable $callback): void { - Runtime::enableCoroutine(SWOOLE_HOOK_ALL); - foreach ($this->collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { continue; } - $sum = 0; - $nextDocument = null; - $collectionCount = $this->projectDB->count($collection['$id']); Console::log('Migrating Collection ' . $collection['$id'] . ':'); - do { - $queries = [Query::limit($this->limit)]; - if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); + \Co\run(function (array $collection, callable $callback) { + foreach ($this->documentsIterator($collection['$id']) as $document) { + go(function (Document $document, callable $callback) { + if (empty($document->getId()) || empty($document->getCollection())) { + return; + } + + $old = $document->getArrayCopy(); + $new = call_user_func($callback, $document); + + if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) { + return; + } + + try { + $new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document); + } catch (\Throwable $th) { + Console::error('Failed to update document: ' . $th->getMessage()); + return; + } + }, $document, $callback); } - $documents = $this->projectDB->find($collection['$id'], $queries); - $count = count($documents); - $sum += $count; - - Console::log($sum . ' / ' . $collectionCount); - - \Co\run(function (array $documents, callable $callback) { - foreach ($documents as $document) { - go(function (Document $document, callable $callback) { - if (empty($document->getId()) || empty($document->getCollection())) { - return; - } - - $old = $document->getArrayCopy(); - $new = call_user_func($callback, $document); - - if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) { - return; - } - - try { - $new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document); - } catch (\Throwable $th) { - Console::error('Failed to update document: ' . $th->getMessage()); - return; - - if ($document && $new->getId() !== $document->getId()) { - throw new Exception('Duplication Error'); - } - } - }, $document, $callback); - } - }, $documents, $callback); - - if ($count !== $this->limit) { - $nextDocument = null; - } else { - $nextDocument = end($documents); - } - } while (!is_null($nextDocument)); + }, $collection, $callback); } } + /** + * Provides an iterator for all documents on a collection. + * + * @param string $collectionId + * @return iterable + * @throws \Exception + */ + public function documentsIterator(string $collectionId): iterable + { + $sum = 0; + $nextDocument = null; + $collectionCount = $this->projectDB->count($collectionId); + + do { + $queries = [Query::limit($this->limit)]; + if ($nextDocument !== null) { + $queries[] = Query::cursorAfter($nextDocument); + } + $documents = $this->projectDB->find($collectionId, $queries); + $count = count($documents); + $sum += $count; + + Console::log($sum . ' / ' . $collectionCount); + foreach ($documents as $document) { + yield $document; + } + + if ($count !== $this->limit) { + $nextDocument = null; + } else { + $nextDocument = end($documents); + } + } while (!is_null($nextDocument)); + } + /** * Checks 2 arrays for differences. * @@ -265,6 +266,8 @@ abstract class Migration } $attribute = $attributes[$attributeKey]; + $filters = $attribute['filters'] ?? []; + $default = $attribute['default'] ?? null; $database->createAttribute( collection: $collectionId, @@ -272,12 +275,12 @@ abstract class Migration type: $attribute['type'], size: $attribute['size'], required: $attribute['required'] ?? false, - default: $attribute['default'] ?? null, + default: in_array('json', $filters) ? json_encode($default) : $default, signed: $attribute['signed'] ?? false, array: $attribute['array'] ?? false, format: $attribute['format'] ?? '', formatOptions: $attribute['formatOptions'] ?? [], - filters: $attribute['filters'] ?? [], + filters: $filters, ); } @@ -287,13 +290,15 @@ abstract class Migration * @param \Utopia\Database\Database $database * @param string $collectionId * @param string $indexId + * @param string|null $from * @return void * @throws \Exception * @throws \Utopia\Database\Exception\Duplicate * @throws \Utopia\Database\Exception\Limit */ - public function createIndexFromCollection(Database $database, string $collectionId, string $indexId): void + public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void { + $from ??= $collectionId; $collection = Config::getParam('collections', [])[$collectionId] ?? null; if (is_null($collection)) { diff --git a/src/Appwrite/Migration/Version/V12.php b/src/Appwrite/Migration/Version/V12.php deleted file mode 100644 index 686566beeb..0000000000 --- a/src/Appwrite/Migration/Version/V12.php +++ /dev/null @@ -1,617 +0,0 @@ -project->getAttribute('name') . ' (' . $this->project->getId() . ')'); - - $this->pdo = $register->get('db'); - - Console::info('Migrating Project Schemas'); - $this->migrateProjectSchema($this->project->getId()); - - /** - * Switch to migrated Console Project - */ - if ($this->project->getId() === 'console') { - $this->consoleDB->setNamespace('_console'); - $this->projectDB->setNamespace('_console'); - } - - Console::info('Migrating Permissions'); - $this->fixPermissions(); - Console::info('Migrating Collections'); - $this->migrateCustomCollections(); - $this->fixCollections(); - Console::info('Migrating Documents'); - $this->forEachDocument([$this, 'fixDocument']); - } - - /** - * Migrate Project Tables. - * - * @param string $projectId - * @return void - * @throws \Exception - * @throws \PDOException - */ - private function migrateProjectSchema(string $projectId): void - { - /** - * Remove empty generated Console Project. - */ - if ($this->consoleDB->getNamespace() === '_project_console' && $projectId === 'console') { - $all = ['_console_bucket_1', '_console_bucket_1_perms']; - foreach ($this->collections as $collection) { - $all[] = "_{$projectId}_{$collection['$id']}"; - $all[] = "_{$projectId}_{$collection['$id']}_perms"; - } - $this->pdo->prepare('DROP TABLE IF EXISTS ' . implode(', ', $all) . ';')->execute(); - } elseif ($this->projectDB->getNamespace() === '_console') { - return; - } - - /** - * Rename Database Tables. - */ - foreach ($this->collections as $collection) { - $id = $collection['$id']; - - /** - * Skip new tables that don't exists on old schema. - */ - if (in_array($id, ['buckets', 'deployments', 'builds'])) { - continue; - } - - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_project_{$projectId}_{$id}` RENAME TO `_{$projectId}_{$id}`")->execute(); - $this->pdo->prepare("CREATE TABLE IF NOT EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$projectId}_{$id}_perms` ( - `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, - `_type` VARCHAR(12) NOT NULL, - `_permission` VARCHAR(255) NOT NULL, - `_document` VARCHAR(255) NOT NULL, - PRIMARY KEY (`_id`), - UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`), - INDEX `_index2` (`_permission`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute(); - } - } - - /** - * Migrate all Collection Structure. - * - * @return void - */ - protected function fixCollections(): void - { - foreach ($this->collections as $collection) { - $id = $collection['$id']; - - /** - * Skip new tables that don't exists on old schema. - */ - if (in_array($id, ['buckets', 'deployments', 'builds'])) { - continue; - } - Console::log("- {$id}"); - switch ($id) { - case 'sessions': - try { - /** - * Rename providerToken to providerAccessToken - */ - $this->projectDB->renameAttribute($id, 'providerToken', 'providerAccessToken'); - } catch (\Throwable $th) { - Console::warning("'providerAccessToken' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create providerRefreshToken - */ - $this->projectDB->createAttribute(collection: $id, id: 'providerRefreshToken', type: Database::VAR_STRING, size: 16384, signed: true, required: false, filters: ['encrypt']); - } catch (\Throwable $th) { - Console::warning("'providerRefreshToken' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create providerAccessTokenExpiry - */ - $this->projectDB->createAttribute(collection: $id, id: 'providerAccessTokenExpiry', type: Database::VAR_INTEGER, size: 0, required: false); - } catch (\Throwable $th) { - Console::warning("'providerAccessTokenExpiry' from {$id}: {$th->getMessage()}"); - } - break; - - case 'memberships': - try { - /** - * Add search attribute and index to memberships. - */ - $this->projectDB->createAttribute(collection: $id, id: 'search', type: Database::VAR_STRING, size: 16384, required: false); - $this->projectDB->createIndex(collection: $id, id: '_key_search', type: Database::INDEX_FULLTEXT, attributes: ['search']); - } catch (\Throwable $th) { - Console::warning("'search' from {$id}: {$th->getMessage()}"); - } - break; - - case 'files': - /** - * Create bucket table if not exists. - */ - $this->createCollection('buckets'); - - if (!$this->projectDB->findOne('buckets', [Query::equal('$id', ['default'])])) { - $this->projectDB->createDocument('buckets', new Document([ - '$id' => ID::custom('default'), - '$collection' => ID::custom('buckets'), - 'dateCreated' => \time(), - 'dateUpdated' => \time(), - 'name' => 'Default', - 'permission' => 'file', - 'maximumFileSize' => (int) App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB - 'allowedFileExtensions' => [], - 'enabled' => true, - 'encryption' => true, - 'antivirus' => true, - '$read' => ['role:all'], - '$write' => ['role:all'], - 'search' => 'buckets Default', - ])); - $this->createCollection('files', 'bucket_1'); - - /** - * Migrate all files to default Bucket. - */ - $nextDocument = null; - do { - $queries = [Query::limit($this->limit)]; - if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); - } - $documents = $this->projectDB->find('files', $queries); - $count = count($documents); - \Co\run(function (array $documents) { - foreach ($documents as $document) { - go(function (Document $document) { - /** - * Update File Path - */ - $path = "/storage/uploads/app-{$this->project->getId()}"; - $new = str_replace($path, "{$path}/default", $document->getAttribute('path')); - $document->setAttribute('path', $new); - - /** - * Populate search string from Migration to 0.12. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'name'], $document)); - } - - /** - * Set new values. - */ - $document - ->setAttribute('bucketId', 'default') - ->setAttribute('chunksTotal', 1) - ->setAttribute('chunksUploaded', 1); - - $this->projectDB->createDocument('bucket_1', $document); - }, $document); - } - }, $documents); - - if ($count !== $this->limit) { - $nextDocument = null; - } else { - $nextDocument = end($documents); - $nextDocument->setAttribute('$collection', 'files'); - } - } while (!is_null($nextDocument)); - - /** - * Rename folder on volumes. - */ - $path = "/storage/uploads/app-{$this->project->getId()}"; - - if (is_dir("{$path}/")) { - mkdir("/storage/uploads/app-{$this->project->getId()}/default"); - - foreach (new \DirectoryIterator($path) as $fileinfo) { - if ($fileinfo->isDir() && !$fileinfo->isDot() && $fileinfo->getFilename() !== 'default') { - rename("{$path}/{$fileinfo->getFilename()}", "{$path}/default/{$fileinfo->getFilename()}"); - } - } - } - } - - break; - - case 'functions': - try { - /** - * Rename tag to deployment - */ - $this->projectDB->renameAttribute($id, 'tag', 'deployment'); - } catch (\Throwable $th) { - Console::warning("'deployment' from {$id}: {$th->getMessage()}"); - } - - /** - * Create deployments table if not exists. - */ - $this->createCollection('deployments'); - - /** - * Create builds table if not exists. - */ - $this->createCollection('builds'); - - break; - - case 'executions': - try { - /** - * Rename tag to deployment - */ - $this->projectDB->renameAttribute($id, 'tagId', 'deploymentId'); - } catch (\Throwable $th) { - Console::warning("'deploymentId' from {$id}: {$th->getMessage()}"); - } - - try { - /** - * Create statusCode - */ - $this->projectDB->createAttribute(collection: $id, id: 'statusCode', type: Database::VAR_INTEGER, size: 0, required: false); - } catch (\Throwable $th) { - Console::warning("'statusCode' from {$id}: {$th->getMessage()}"); - } - - break; - - case 'teams': - try { - /** - * Rename tag to deployment - */ - $this->projectDB->renameAttribute($id, 'sum', 'total'); - } catch (\Throwable $th) { - Console::warning("'total' from {$id}: {$th->getMessage()}"); - } - - break; - } - usleep(100000); - } - } - - /** - * Migrates permissions to dedicated table. - * - * @param \Utopia\Database\Document $document - * @param string $internalId - * @return void - * @throws \Exception - * @throws \PDOException - */ - protected function migratePermissionsToDedicatedTable(string $collection, Document $document): void - { - $sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$collection}` WHERE _uid = {$this->pdo->quote($document->getid())}"; - $stmt = $this->pdo->prepare($sql); - $stmt->execute(); - - $permissions = $stmt->fetch(); - - $read = json_decode($permissions['_read'] ?? null) ?? []; - $write = json_decode($permissions['_write'] ?? null) ?? []; - - $permissions = []; - foreach ($read as $permission) { - $permissions[] = "('read', '{$permission}', '{$document->getId()}')"; - } - - foreach ($write as $permission) { - $permissions[] = "('write', '{$permission}', '{$document->getId()}')"; - } - - if (!empty($permissions)) { - $queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$collection}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions); - $stmtPermissions = $this->pdo->prepare($queryPermissions); - $stmtPermissions->execute(); - } - } - - /** - * Migrates all user's database collections. - * - * @return void - * @throws \Exception - */ - protected function migrateCustomCollections(): void - { - $nextCollection = null; - - do { - $queries = [Query::limit($this->limit)]; - if ($nextCollection !== null) { - $queries[] = Query::cursorAfter($nextCollection); - } - $documents = $this->projectDB->find('collections', $queries); - $count = count($documents); - - \Co\run(function (array $documents) { - foreach ($documents as $document) { - go(function (Document $collection) { - $id = $collection->getId(); - $projectId = $this->project->getId(); - $internalId = $collection->getInternalId(); - - if ($this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), "collection_{$internalId}")) { - return; - } - Console::log("- {$id} ({$collection->getAttribute('name')})"); - - /** - * Rename user's colletion table schema - */ - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_project_{$projectId}_collection_{$id}` RENAME TO `_{$projectId}_collection_{$internalId}`")->execute(); - $this->pdo->prepare("CREATE TABLE IF NOT EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$projectId}_collection_{$internalId}_perms` ( - `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, - `_type` VARCHAR(12) NOT NULL, - `_permission` VARCHAR(255) NOT NULL, - `_document` VARCHAR(255) NOT NULL, - PRIMARY KEY (`_id`), - UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`), - INDEX `_index2` (`_permission`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute(); - - /** - * Update metadata table. - */ - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$projectId}__metadata` - SET - _uid = 'collection_{$internalId}', - name = 'collection_{$internalId}' - WHERE _uid = 'collection_{$id}'; - ")->execute(); - - - $nextDocument = null; - - do { - $queries = [Query::limit($this->limit)]; - if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); - } - $documents = $this->projectDB->find('collection_' . $internalId, $queries); - $count = count($documents); - - foreach ($documents as $document) { - go(function (Document $document, string $internalId) { - $this->migratePermissionsToDedicatedTable("collection_{$internalId}", $document); - }, $document, $internalId); - } - - if ($count !== $this->limit) { - $nextDocument = null; - } else { - $nextDocument = end($documents); - } - } while (!is_null($nextDocument)); - - /** - * Remove _read and _write columns - */ - $this->pdo->prepare(" - ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_collection_{$internalId}` - DROP COLUMN _read, - DROP COLUMN _write - ")->execute(); - }, $document); - } - }, $documents); - - if ($count !== $this->limit) { - $nextCollection = null; - } else { - $nextCollection = end($documents); - } - } while (!is_null($nextCollection)); - } - - /** - * Migrate all Permission to new System with dedicated Table. - * - * @return void - * @throws \Exception - */ - protected function fixPermissions() - { - foreach ($this->collections as $collection) { - $id = $collection['$id']; - - /** - * Skip new tables that don't exists on old schema. - */ - if (in_array($id, ['buckets', 'deployments', 'builds'])) { - continue; - } - /** - * Check if permissions have already been migrated. - */ - try { - $stmtCheck = $this->pdo->prepare("SHOW COLUMNS from `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$id}` LIKE '_read'"); - $stmtCheck->execute(); - - if (empty($stmtCheck->fetchAll())) { - continue; - } - } catch (\Throwable $th) { - if ($th->getCode() === "42S02") { - continue; - } - throw $th; - } - - - Console::log("- {$collection['$id']}"); - $nextDocument = null; - - do { - $queries = [Query::limit($this->limit)]; - if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); - } - $documents = $this->projectDB->find($id, $queries); - $count = count($documents); - - \Co\run(function (array $documents) { - foreach ($documents as $document) { - go(function (Document $document) { - $this->migratePermissionsToDedicatedTable($document->getCollection(), $document); - }, $document); - } - }, $documents); - - if ($count !== $this->limit) { - $nextDocument = null; - } else { - $nextDocument = end($documents); - } - } while (!is_null($nextDocument)); - - /** - * Remove _read and _write columns - */ - $this->pdo->prepare(" - ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$id}` - DROP COLUMN _read, - DROP COLUMN _write - ")->execute(); - } - - /** - * Timeout to give MariaDB some room to breath - */ - usleep(100000); - } - - /** - * Fix run on each document - * - * @param \Utopia\Database\Document $document - * @return \Utopia\Database\Document - */ - protected function fixDocument(Document $document) - { - switch ($document->getCollection()) { - case 'projects': - /** - * Bump Project version number. - */ - $document->setAttribute('version', '0.13.0'); - - /** - * Populate search string from Migration to 0.12. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'name'], $document)); - } - - break; - - case 'users': - /** - * Populate search string from Migration to 0.12. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'email', 'name'], $document)); - } - - break; - - case 'teams': - /** - * Populate search string from Migration to 0.12. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'name'], $document)); - } - - break; - - case 'functions': - $document->setAttribute('deployment', null); - - /** - * Populate search string from Migration to 0.12. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'name', 'runtime'], $document)); - } - - break; - - case 'executions': - /** - * Populate search string from Migration to 0.12. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'functionId'], $document)); - } - - break; - - case 'memberships': - /** - * Populate search string. - */ - if (empty($document->getAttribute('search'))) { - $document->setAttribute('search', $this->buildSearchAttribute(['$id', 'userId'], $document)); - } - - break; - - case 'sessions': - $document - ->setAttribute('providerRefreshToken', '') - ->setAttribute('providerAccessTokenExpiry', 0) - ->setAttribute('providerAccessToken', $document->getAttribute('providerToken', '')) - ->removeAttribute('providerToken'); - - break; - } - - return $document; - } - - /** - * Builds a search string for a fulltext index. - * - * @param array $values - * @param Document $document - * @return string - */ - private function buildSearchAttribute(array $values, Document $document): string - { - $values = array_filter(array_map(fn (string $value) => $document->getAttribute($value) ?? '', $values)); - - return implode(' ', $values); - } -} diff --git a/src/Appwrite/Migration/Version/V13.php b/src/Appwrite/Migration/Version/V13.php deleted file mode 100644 index d204be84f3..0000000000 --- a/src/Appwrite/Migration/Version/V13.php +++ /dev/null @@ -1,337 +0,0 @@ -project->getAttribute('name') . ' (' . $this->project->getId() . ')'); - Console::info('Migrating Collections'); - $this->migrateCollections(); - Console::info('Migrating Documents'); - $this->forEachDocument([$this, 'fixDocument']); - } - - /** - * Migrate all Collections. - * - * @return void - */ - protected function migrateCollections(): void - { - foreach ($this->collections as $collection) { - $id = $collection['$id']; - - Console::log("- {$id}"); - switch ($id) { - case 'projects': - try { - /** - * Rename providers to authProviders. - */ - $this->projectDB->renameAttribute($id, 'providers', 'authProviders'); - } catch (\Throwable $th) { - Console::warning("'providers' from {$id}: {$th->getMessage()}"); - } - break; - case 'users': - try { - /** - * Recreate sessions for new subquery. - */ - $this->projectDB->deleteAttribute($id, 'sessions'); - $this->projectDB->createAttribute( - collection: $id, - id: 'sessions', - required: false, - type: Database::VAR_STRING, - format: '', - size: 16384, - filters: ['subQuerySessions'] - ); - } catch (\Throwable $th) { - Console::warning("'sessions' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Recreate tokens for new subquery. - */ - $this->projectDB->deleteAttribute($id, 'tokens'); - $this->projectDB->createAttribute( - collection: $id, - id: 'tokens', - required: false, - type: Database::VAR_STRING, - format: '', - size: 16384, - filters: ['subQueryTokens'] - ); - } catch (\Throwable $th) { - Console::warning("'tokens' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Recreate memberships for new subquery. - */ - $this->projectDB->deleteAttribute($id, 'memberships'); - $this->projectDB->createAttribute( - collection: $id, - id: 'memberships', - required: false, - type: Database::VAR_STRING, - format: '', - size: 16384, - filters: ['subQueryMemberships'] - ); - } catch (\Throwable $th) { - Console::warning("'memberships' from {$id}: {$th->getMessage()}"); - } - break; - case 'sessions': - try { - /** - * Add new index for users. - */ - $this->projectDB->createIndex(collection: $id, id: '_key_user', type: Database::INDEX_KEY, attributes: ['userId'], orders: [Database::ORDER_ASC]); - } catch (\Throwable $th) { - Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); - } - break; - case 'builds': - try { - /** - * Increase stdout size. - */ - $this->projectDB->updateAttribute($id, 'stdout', size: 1_000_000); - } catch (\Throwable $th) { - Console::warning("'stdout' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Increase stderr size. - */ - $this->projectDB->updateAttribute($id, 'stderr', size: 1_000_000); - } catch (\Throwable $th) { - Console::warning("'stderr' from {$id}: {$th->getMessage()}"); - } - break; - case 'executions': - try { - /** - * Rename stdout to response. - * Increase response size. - */ - $this->projectDB->renameAttribute($id, 'stdout', 'response'); - $this->projectDB->updateAttribute($id, 'response', size: 1_000_000); - } catch (\Throwable $th) { - Console::warning("'stdout' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Increase stderr size. - */ - $this->projectDB->updateAttribute($id, 'stderr', size: 1_000_000); - } catch (\Throwable $th) { - Console::warning("'stderr' from {$id}: {$th->getMessage()}"); - } - break; - case 'stats': - try { - /** - * Increase value size ot BIGINT. - */ - $this->projectDB->updateAttribute($id, 'value', size: 8); - } catch (\Throwable $th) { - Console::warning("'size' from {$id}: {$th->getMessage()}"); - } - break; - case 'tokens': - try { - /** - * Create new Tokens collection. - */ - $this->createCollection('tokens'); - } catch (\Throwable $th) { - Console::warning("'tokens': {$th->getMessage()}"); - } - break; - } - usleep(100000); - } - } - - /** - * Fix run on each document - * - * @param \Utopia\Database\Document $document - * @return \Utopia\Database\Document - */ - protected function fixDocument(Document $document) - { - switch ($document->getCollection()) { - case 'projects': - /** - * Bump Project version number. - */ - $document->setAttribute('version', '0.14.0'); - - break; - - case 'functions': - /** - * Migrate events. - */ - if (!empty($document->getAttribute('events'))) { - $document->setAttribute('events', $this->migrateEvents($document->getAttribute('events'))); - } - - break; - - case 'webhooks': - /** - * Migrate events. - */ - if (!empty($document->getAttribute('events'))) { - $document->setAttribute('events', $this->migrateEvents($document->getAttribute('events'))); - } - - break; - - case 'users': - /** - * Remove deleted users. - */ - if ($document->getAttribute('deleted', false) === true) { - $this->projectDB->deleteDocument('users', $document->getId()); - } - break; - } - - return $document; - } - - public function migrateEvents(array $events): array - { - return array_filter(array_unique(array_map(function ($event) { - if (!in_array($event, $this->events)) { - return $event; - } - $parts = \explode('.', $event); - $first = array_shift($parts); - switch ($first) { - case 'account': - case 'users': - $first = 'users'; - - switch ($parts[0]) { - case 'recovery': - case 'sessions': - case 'verification': - $second = array_shift($parts); - return 'users.*.' . $second . '.*.' . implode('.', $parts); - - default: - return 'users.*.' . implode('.', $parts); - } - case 'functions': - switch ($parts[0]) { - case 'deployments': - case 'executions': - $second = array_shift($parts); - return 'functions.*.' . $second . '.*.' . implode('.', $parts); - - default: - return 'functions.*.' . implode('.', $parts); - } - case 'teams': - switch ($parts[0]) { - case 'memberships': - $second = array_shift($parts); - return 'teams.*.' . $second . '.*.' . implode('.', $parts); - - default: - return 'teams.*.' . implode('.', $parts); - } - case 'storage': - $second = array_shift($parts); - switch ($second) { - case 'buckets': - return 'buckets.*.' . implode('.', $parts); - case 'files': - return 'buckets.*.' . $second . '.*.' . implode('.', $parts); - } // intentional fallthrough - case 'database': - $second = array_shift($parts); - switch ($second) { - case 'collections': - return 'collections.*.' . implode('.', $parts); - case 'documents': - case 'indexes': - case 'attributes': - return 'collections.*.' . $second . '.*.' . implode('.', $parts); - } - } - return ''; - }, $events))); - } -} diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php deleted file mode 100644 index e74788d102..0000000000 --- a/src/Appwrite/Migration/Version/V14.php +++ /dev/null @@ -1,806 +0,0 @@ -pdo = $register->get('db'); - - if ($this->project->getId() === 'console' && $this->project->getInternalId() !== 'console') { - return; - } - - /** - * Disable SubQueries for Speed. - */ - foreach (['subQueryAttributes', 'subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships'] as $name) { - Database::addFilter($name, fn () => null, fn () => []); - } - - Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); - Console::info('Migrating Collections'); - $this->migrateCollections(); - Console::info('Create Default Database Layer'); - $this->createDatabaseLayer(); - if ($this->project->getId() !== 'console') { - Console::info('Migrating Database Collections'); - $this->migrateCustomCollections(); - } - Console::info('Migrating Documents'); - $this->forEachDocument([$this, 'fixDocument']); - } - - /** - * Creates the default Database for existing Projects. - * - * @return void - * @throws \Throwable - */ - public function createDatabaseLayer(): void - { - try { - if (!$this->projectDB->exists('databases')) { - $this->createCollection('databases'); - } - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - - if ($this->project->getInternalId() === 'console') { - return; - } - - try { - $this->projectDB->createDocument('databases', new Document([ - '$id' => ID::custom('default'), - 'name' => 'Default', - 'search' => 'default Default' - ])); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - } - - /** - * Migrates all Files. - * - * @param \Utopia\Database\Document $bucket - * @return void - * @throws \Exception - */ - protected function migrateBucketFiles(Document $bucket): void - { - $nextFile = null; - do { - $queries = [Query::limit($this->limit)]; - if ($nextFile !== null) { - $queries[] = Query::cursorAfter($nextFile); - } - $documents = $this->projectDB->find("bucket_{$bucket->getInternalId()}", $queries); - $count = count($documents); - - foreach ($documents as $document) { - go(function (Document $bucket, Document $document) { - Console::log("Migrating File {$document->getId()}"); - try { - /** - * Migrate $createdAt. - */ - if (empty($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - $this->projectDB->updateDocument("bucket_{$bucket->getInternalId()}", $document->getId(), $document); - } - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - }, $bucket, $document); - } - - if ($count !== $this->limit) { - $nextFile = null; - } else { - $nextFile = end($documents); - } - } while (!is_null($nextFile)); - } - - /** - * Migrates all Database Collections. - * @return void - * @throws \Exception - */ - protected function migrateCustomCollections(): void - { - try { - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections` RENAME TO `_{$this->project->getInternalId()}_database_1`")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - try { - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections_perms` RENAME TO `_{$this->project->getInternalId()}_database_1_perms`")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - - /** - * Update metadata table. - */ - try { - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` - SET - _uid = 'database_1', - name = 'database_1' - WHERE _uid = 'collections'; - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - - try { - /** - * Add Database ID for Collections. - */ - $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseId', 'collections'); - - /** - * Add Database Internal ID for Collections. - */ - $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseInternalId', 'collections'); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - - $nextCollection = null; - - do { - $queries = [Query::limit($this->limit)]; - if ($nextCollection !== null) { - $queries[] = Query::cursorAfter($nextCollection); - } - $documents = $this->projectDB->find('database_1', $queries); - $count = count($documents); - - \Co\run(function (array $documents) { - foreach ($documents as $document) { - go(function (Document $collection) { - $id = $collection->getId(); - $internalId = $collection->getInternalId(); - - Console::log("- {$id} ({$collection->getAttribute('name')})"); - - try { - /** - * Rename user's colletion table schema - */ - $this->createNewMetaData("collection_{$internalId}", "database_1_collection_{$internalId}"); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - - try { - /** - * Update metadata table. - */ - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` - SET - _uid = 'database_1_collection_{$internalId}', - name = 'database_1_collection_{$internalId}' - WHERE _uid = 'collection_{$internalId}'; - ")->execute(); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - - try { - /** - * Update internal ID's. - */ - $collection - ->setAttribute('databaseId', 'default') - ->setAttribute('databaseInternalId', '1'); - $this->projectDB->updateDocument('database_1', $collection->getId(), $collection); - } catch (\Throwable $th) { - Console::warning($th->getMessage()); - } - /** - * Migrate Attributes - */ - $this->migrateAttributesAndCollections('attributes', $collection); - /** - * Migrate Indexes - */ - $this->migrateAttributesAndCollections('indexes', $collection); - }, $document); - } - }, $documents); - - if ($count !== $this->limit) { - $nextCollection = null; - } else { - $nextCollection = end($documents); - } - } while (!is_null($nextCollection)); - } - - protected function migrateAttributesAndCollections(string $type, Document $collection): void - { - /** - * Offset pagination instead of cursor, since documents are re-created! - */ - $offset = 0; - $attributesCount = $this->projectDB->count($type, queries: [Query::equal('collectionId', [$collection->getId()])]); - - do { - $queries = [ - Query::limit($this->limit), - Query::offset($offset), - Query::equal('collectionId', [$collection->getId()]), - ]; - $documents = $this->projectDB->find($type, $queries); - $offset += $this->limit; - - foreach ($documents as $document) { - go(function (Document $document, string $internalId, string $type) { - try { - /** - * Skip already migrated Documents. - */ - if (!is_null($document->getAttribute('databaseId'))) { - return; - } - /** - * Add Internal ID 'collectionInternalId' for Subqueries. - */ - $document->setAttribute('collectionInternalId', $internalId); - /** - * Add Internal ID 'databaseInternalId' for Subqueries. - */ - $document->setAttribute('databaseInternalId', '1'); - /** - * Add Internal ID 'databaseId'. - */ - $document->setAttribute('databaseId', 'default'); - - /** - * Re-create Attribute. - */ - $this->projectDB->deleteDocument($document->getCollection(), $document->getId()); - $this->projectDB->createDocument($document->getCollection(), $document->setAttribute('$id', "1_{$internalId}_{$document->getAttribute('key')}")); - } catch (\Throwable $th) { - Console::error("Failed to {$type} document: " . $th->getMessage()); - } - }, $document, $collection->getInternalId(), $type); - } - } while ($offset < $attributesCount); - } - - /** - * Migrate all Collections. - * - * @return void - */ - protected function migrateCollections(): void - { - foreach ($this->collections as $collection) { - $id = $collection['$id']; - - Console::log("- {$id}"); - - $this->createNewMetaData($id); - - $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); - - switch ($id) { - case 'attributes': - case 'indexes': - try { - /** - * Create 'databaseInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'databaseId'); - } catch (\Throwable $th) { - Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create 'databaseInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'databaseInternalId'); - } catch (\Throwable $th) { - Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); - } - - try { - /** - * Create 'collectionInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'collectionInternalId'); - } catch (\Throwable $th) { - Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); - } - - try { - /** - * Re-Create '_key_collection' index - */ - @$this->projectDB->deleteIndex($id, '_key_collection'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_db_collection'); - } catch (\Throwable $th) { - Console::warning("'_key_collection' from {$id}: {$th->getMessage()}"); - } - - break; - case 'projects': - try { - /** - * Create 'teamInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); - } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); - } - - break; - case 'platforms': - case 'domains': - try { - /** - * Create 'projectInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); - } catch (\Throwable $th) { - Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_project' index - */ - @$this->projectDB->deleteIndex($id, '_key_project'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); - } catch (\Throwable $th) { - Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); - } - - break; - case 'keys': - try { - /** - * Create 'projectInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); - } catch (\Throwable $th) { - Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create 'expire' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'expire'); - } catch (\Throwable $th) { - Console::warning("'expire' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_project' index - */ - @$this->projectDB->deleteIndex($id, '_key_project'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); - } catch (\Throwable $th) { - Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); - } - - break; - case 'webhooks': - try { - /** - * Create 'signatureKey' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'signatureKey'); - } catch (\Throwable $th) { - Console::warning("'signatureKey' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create 'projectInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); - } catch (\Throwable $th) { - Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_project' index - */ - @$this->projectDB->deleteIndex($id, '_key_project'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); - } catch (\Throwable $th) { - Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); - } - - break; - case 'users': - try { - /** - * Create 'phone' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'phone'); - } catch (\Throwable $th) { - Console::warning("'phone' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create 'phoneVerification' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'phoneVerification'); - } catch (\Throwable $th) { - Console::warning("'phoneVerification' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create '_key_phone' index - */ - $this->createIndexFromCollection($this->projectDB, $id, '_key_phone'); - } catch (\Throwable $th) { - Console::warning("'_key_phone' from {$id}: {$th->getMessage()}"); - } - - break; - case 'tokens': - case 'sessions': - try { - /** - * Create 'userInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); - } catch (\Throwable $th) { - Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_user' index - */ - @$this->projectDB->deleteIndex($id, '_key_user'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); - } catch (\Throwable $th) { - Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); - } - - break; - case 'memberships': - try { - /** - * Create 'teamInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); - } catch (\Throwable $th) { - Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Create 'userInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); - } catch (\Throwable $th) { - Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_unique' index - */ - @$this->projectDB->deleteIndex($id, '_key_unique'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_unique'); - } catch (\Throwable $th) { - Console::warning("'_key_unique' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_team' index - */ - @$this->projectDB->deleteIndex($id, '_key_team'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_team'); - } catch (\Throwable $th) { - Console::warning("'_key_team' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_user' index - */ - @$this->projectDB->deleteIndex($id, '_key_user'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); - } catch (\Throwable $th) { - Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); - } - break; - } - usleep(50000); - } - } - - /** - * Fix run on each document - * - * @param \Utopia\Database\Document $document - * @return \Utopia\Database\Document - */ - protected function fixDocument(Document $document) - { - switch ($document->getCollection()) { - case 'projects': - /** - * Bump Project version number. - */ - $document->setAttribute('version', '0.15.0'); - - if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) { - $internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId(); - $document->setAttribute('teamInternalId', $internalId); - } - - break; - case 'keys': - /** - * Add new 'expire' attribute and default to never (0). - */ - if (is_null($document->getAttribute('expire'))) { - $document->setAttribute('expire', 0); - } - /** - * Add Internal ID 'projectId' for Subqueries. - */ - if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { - $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); - $document->setAttribute('projectInternalId', $internalId); - } - - break; - case 'audit': - /** - * Add Database Layer to collection resource. - */ - if (str_starts_with($document->getAttribute('resource'), 'collection/')) { - $document - ->setAttribute('resource', "database/default/{$document->getAttribute('resource')}") - ->setAttribute('event', "databases.default.{$document->getAttribute('event')}"); - } - - if (str_starts_with($document->getAttribute('resource'), 'document/')) { - $collectionId = explode('.', $document->getAttribute('event'))[1]; - $document - ->setAttribute('resource', "database/default/collection/{$collectionId}/{$document->getAttribute('resource')}") - ->setAttribute('event', "databases.default.{$document->getAttribute('event')}"); - } - - break; - case 'stats': - /** - * Add Database Layer to stats metric. - */ - if (str_starts_with($document->getAttribute('metric'), 'database.')) { - $metric = ltrim($document->getAttribute('metric'), 'database.'); - $document->setAttribute('metric', "databases.default.{$metric}"); - } - - break; - case 'webhooks': - /** - * Add new 'signatureKey' attribute and generate a random value. - */ - if (empty($document->getAttribute('signatureKey'))) { - $document->setAttribute('signatureKey', \bin2hex(\random_bytes(64))); - } - /** - * Add Internal ID 'projectId' for Subqueries. - */ - if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { - $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); - $document->setAttribute('projectInternalId', $internalId); - } - - break; - case 'domains': - /** - * Add Internal ID 'projectId' for Subqueries. - */ - if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { - $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); - $document->setAttribute('projectInternalId', $internalId); - } - - break; - case 'tokens': - case 'sessions': - /** - * Add Internal ID 'userId' for Subqueries. - */ - if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { - $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); - $document->setAttribute('userInternalId', $internalId); - } - - break; - case 'memberships': - /** - * Add Internal ID 'userId' for Subqueries. - */ - if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { - $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); - $document->setAttribute('userInternalId', $internalId); - } - /** - * Add Internal ID 'teamId' for Subqueries. - */ - if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) { - $internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId(); - $document->setAttribute('teamInternalId', $internalId); - } - - break; - case 'platforms': - /** - * Migrate dateCreated to $createdAt. - */ - if (empty($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } - /** - * Migrate dateUpdated to $updatedAt. - */ - if (empty($document->getUpdatedAt())) { - $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); - } - /** - * Add Internal ID 'projectId' for Subqueries. - */ - if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { - $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); - $document->setAttribute('projectInternalId', $internalId); - } - - break; - case 'buckets': - /** - * Migrate dateCreated to $createdAt. - */ - if (empty($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } - /** - * Migrate dateUpdated to $updatedAt. - */ - if (empty($document->getUpdatedAt())) { - $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); - } - - /** - * Migrate all Storage Buckets to use Internal ID. - */ - $internalId = $this->projectDB->getDocument('buckets', $document->getId())->getInternalId(); - $this->createNewMetaData("bucket_{$internalId}"); - - /** - * Migrate all Storage Bucket Files. - */ - $this->migrateBucketFiles($document); - - break; - case 'users': - /** - * Set 'phoneVerification' to false if not set. - */ - if (is_null($document->getAttribute('phoneVerification'))) { - $document->setAttribute('phoneVerification', false); - } - - break; - case 'functions': - /** - * Migrate dateCreated to $createdAt. - */ - if (empty($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } - /** - * Migrate dateUpdated to $updatedAt. - */ - if (empty($document->getUpdatedAt())) { - $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); - } - - break; - case 'deployments': - case 'executions': - case 'teams': - /** - * Migrate dateCreated to $createdAt. - */ - if (empty($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } - - break; - } - - return $document; - } - - /** - * Creates new metadata that was introduced for a collection and enforces the Internal ID. - * - * @param string $id - * @return void - */ - protected function createNewMetaData(string $id, string $to = null): void - { - $to ??= $id; - /** - * Skip files collection. - */ - if (in_array($id, ['files', 'databases'])) { - return; - } - - try { - /** - * Replace project UID with Internal ID. - */ - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$to}`")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Replace project UID with Internal ID on permissions table. - */ - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$to}_perms`")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Add _createdAt attribute. - */ - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Add _updatedAt attribute. - */ - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Create index for _createdAt. - */ - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$to}` (`_createdAt`)")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Create index for _updatedAt. - */ - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$to}` (`_updatedAt`)")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - } -} diff --git a/src/Appwrite/Migration/Version/V15.php b/src/Appwrite/Migration/Version/V15.php new file mode 100644 index 0000000000..27cd32bcf0 --- /dev/null +++ b/src/Appwrite/Migration/Version/V15.php @@ -0,0 +1,1489 @@ + + */ + protected array $providers; + + public function execute(): void + { + global $register; + $this->pdo = $register->get('db'); + + /** + * Populate providers. + */ + $this->providers = \array_merge( + ['email', 'anonymous'], + \array_map( + fn ($value) => "oauth-" . $value, + \array_keys(Config::getParam('providers', [])) + ) + ); + + /** + * Disable SubQueries for Performance. + */ + foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subqueryVariables'] as $name) { + Database::addFilter( + $name, + fn () => null, + fn () => [] + ); + } + + Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + Console::info('Migrating Stats'); + $this->migrateStatsMetric('requests', 'project.$all.network.requests'); + $this->migrateStatsMetric('network', 'project.$all.network.bandwidth'); + $this->migrateStatsMetric('executions', 'executions.$all.compute.total'); + $this->migrateStatsMetric('storage.total', 'project.$all.storage.size'); + + Console::info('Migrating Collections'); + $this->migrateCollections(); + Console::info('Migrating Databases'); + $this->migrateDatabases(); + Console::info('Migrating Buckets'); + $this->migrateBuckets(); + Console::info('Migrating Documents'); + $this->forEachDocument([$this, 'fixDocument']); + Console::info("Clean up 'write' Permissions"); + foreach ($this->collections as $collection) { + if ($collection['$collection'] === Database::METADATA) { + $this->removeWritePermissions($collection['$id']); + } + } + } + + /** + * Migrating all Bucket tables. + * + * @return void + * @throws \Exception + * @throws \PDOException + */ + protected function migrateBuckets(): void + { + /** + * Migrating stats for all Buckets. + */ + $this->migrateStatsMetric('storage.files.total', 'files.$all.storage.size'); + $this->migrateStatsMetric('storage.files.count', 'files.$all.count.total'); + $this->migrateStatsMetric('storage.buckets.count', 'buckets.$all.count.total'); + $this->migrateStatsMetric('storage.buckets.create', 'buckets.$all.requests.create'); + $this->migrateStatsMetric('storage.buckets.read', 'buckets.$all.requests.read'); + $this->migrateStatsMetric('storage.buckets.update', 'buckets.$all.requests.update'); + $this->migrateStatsMetric('storage.buckets.delete', 'buckets.$all.requests.delete'); + $this->migrateStatsMetric('storage.files.create', 'files.$all.requests.create'); + $this->migrateStatsMetric('storage.files.read', 'files.$all.requests.read'); + $this->migrateStatsMetric('storage.files.update', 'files.$all.requests.update'); + $this->migrateStatsMetric('storage.files.delete', 'files.$all.requests.delete'); + + foreach ($this->documentsIterator('buckets') as $bucket) { + $bucketTable = "bucket_{$bucket->getInternalId()}"; + + $this->createPermissionsColumn($bucketTable); + $this->migrateDateTimeAttribute($bucketTable, '_createdAt'); + $this->migrateDateTimeAttribute($bucketTable, '_updatedAt'); + + $this->populatePermissionsAttribute( + document: $bucket, + addCreatePermission: true + ); + + if (!is_null($bucket->getAttribute('permission'))) { + $bucket->setAttribute('fileSecurity', $bucket->getAttribute('permissions') === 'document'); + } + + if (is_null($bucket->getAttribute('compression'))) { + $bucket->setAttribute('compression', 'none'); + } + + $this->projectDB->updateDocument('buckets', $bucket->getId(), $bucket); + + /** + * Migrating stats for every Bucket. + */ + $bucketId = $bucket->getId(); + $this->migrateStatsMetric("storage.buckets.$bucketId.files.count", "files.$bucketId.count.total"); + $this->migrateStatsMetric("storage.buckets.$bucketId.files.total", "files.$bucketId.storage.size"); + $this->migrateStatsMetric("storage.buckets.$bucketId.files.create", "files.$bucketId.requests.create"); + $this->migrateStatsMetric("storage.buckets.$bucketId.files.read", "files.$bucketId.requests.read"); + $this->migrateStatsMetric("storage.buckets.$bucketId.files.update", "files.$bucketId.requests.update"); + $this->migrateStatsMetric("storage.buckets.$bucketId.files.delete", "files.$bucketId.requests.delete"); + + Console::info("Migrating Files of {$bucket->getId()} ({$bucket->getAttribute('name')})"); + foreach ($this->documentsIterator($bucketTable) as $file) { + $this->populatePermissionsAttribute( + document: $file, + table: $bucketTable, + addCreatePermission: false + ); + $this->projectDB->updateDocument($bucketTable, $file->getId(), $file); + } + $this->removeWritePermissions($bucketTable); + } + + try { + $this->projectDB->deleteAttribute('buckets', 'permission'); + } catch (\Throwable $th) { + Console::warning("'permissions' from buckets: {$th->getMessage()}"); + } + } + + /** + * Migrating all Database and Collection tables. + * + * @return void + * @throws \Exception + * @throws \PDOException + */ + protected function migrateDatabases(): void + { + /** + * Migrating stats for all Databases. + */ + $this->migrateStatsMetric('databases.count', 'databases.$all.count.total'); + $this->migrateStatsMetric('databases.documents.count', 'documents.$all.count.total'); + $this->migrateStatsMetric('databases.collections.count', 'collections.$all.count.total'); + $this->migrateStatsMetric('databases.create', 'databases.$all.requests.create'); + $this->migrateStatsMetric('databases.read', 'databases.$all.requests.read'); + $this->migrateStatsMetric('databases.update', 'databases.$all.requests.update'); + $this->migrateStatsMetric('databases.delete', 'databases.$all.requests.delete'); + $this->migrateStatsMetric('databases.collections.create', 'collections.$all.requests.create'); + $this->migrateStatsMetric('databases.collections.read', 'collections.$all.requests.read'); + $this->migrateStatsMetric('databases.collections.update', 'collections.$all.requests.update'); + $this->migrateStatsMetric('databases.collections.delete', 'collections.$all.requests.delete'); + $this->migrateStatsMetric('databases.documents.create', 'documents.$all.requests.create'); + $this->migrateStatsMetric('databases.documents.read', 'documents.$all.requests.read'); + $this->migrateStatsMetric('databases.documents.update', 'documents.$all.requests.update'); + $this->migrateStatsMetric('databases.documents.delete', 'documents.$all.requests.delete'); + + /** + * Migrate every Database. + */ + foreach ($this->documentsIterator('databases') as $database) { + $databaseTable = "database_{$database->getInternalId()}"; + $this->createPermissionsColumn($databaseTable); + $this->migrateDateTimeAttribute($databaseTable, '_createdAt'); + $this->migrateDateTimeAttribute($databaseTable, '_updatedAt'); + $this->populatePermissionsAttribute( + document: $database, + table: 'databases', + addCreatePermission: false + ); + + $this->projectDB->updateDocument('databases', $database->getId(), $database); + + try { + $this->createAttributeFromCollection($this->projectDB, $databaseTable, 'documentSecurity', 'collections'); + } catch (\Throwable $th) { + Console::warning("'documentSecurity' from {$databaseTable}: {$th->getMessage()}"); + } + + /** + * Migrating stats for single Databases. + */ + $databaseId = $database->getId(); + $this->migrateStatsMetric("databases.$databaseId.collections.count", "collections.$databaseId.count.total"); + $this->migrateStatsMetric("databases.$databaseId.collections.create", "collections.$databaseId.requests.create"); + $this->migrateStatsMetric("databases.$databaseId.collections.read", "collections.$databaseId.requests.read"); + $this->migrateStatsMetric("databases.$databaseId.collections.update", "collections.$databaseId.requests.update"); + $this->migrateStatsMetric("databases.$databaseId.collections.delete", "collections.$databaseId.requests.delete"); + $this->migrateStatsMetric("databases.$databaseId.documents.count", "documents.$databaseId.count.total"); + $this->migrateStatsMetric("databases.$databaseId.documents.create", "documents.$databaseId.requests.create"); + $this->migrateStatsMetric("databases.$databaseId.documents.read", "documents.$databaseId.requests.read"); + $this->migrateStatsMetric("databases.$databaseId.documents.update", "documents.$databaseId.requests.update"); + $this->migrateStatsMetric("databases.$databaseId.documents.delete", "documents.$databaseId.requests.delete"); + + /** + * Migrate every Collection. + */ + Console::info("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})"); + foreach ($this->documentsIterator($databaseTable) as $collection) { + $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}"; + $this->createPermissionsColumn($collectionTable); + $this->migrateDateTimeAttribute($collectionTable, '_createdAt'); + $this->migrateDateTimeAttribute($collectionTable, '_updatedAt'); + + $this->populatePermissionsAttribute( + document: $collection, + table: $databaseTable, + addCreatePermission: true + ); + + if (!is_null($collection->getAttribute('permission'))) { + $collection->setAttribute('documentSecurity', $collection->getAttribute('permissions') === 'document'); + } + + $this->projectDB->updateDocument($databaseTable, $collection->getId(), $collection); + + /** + * Migrating stats for single Collections. + */ + $collectionId = $collection->getId(); + $this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.count", "documents.{$databaseId}/{$collectionId}.count.total"); + $this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.create", "documents.{$databaseId}/{$collectionId}.requests.create"); + $this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.read", "documents.{$databaseId}/{$collectionId}.requests.read"); + $this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.update", "documents.{$databaseId}/{$collectionId}.requests.update"); + $this->migrateStatsMetric("databases.{$databaseId}.collections.{$collectionId}.documents.delete", "documents.{$databaseId}/{$collectionId}.requests.delete"); + + Console::info("Migrating Documents of {$collection->getId()} ({$collection->getAttribute('name')})"); + $requiredAttributes = array_reduce($collection->getAttribute('attributes', []), function (array $carry, Document $item) { + if ($item->getAttribute('required', false)) { + $carry = array_merge($carry, [ + $item->getAttribute('key') => $item->getAttribute('default') + ]); + } + return $carry; + }, []); + + foreach ($this->documentsIterator($collectionTable) as $document) { + foreach ($document->getAttributes() as $attribute => $default) { + if (array_key_exists($attribute, $requiredAttributes)) { + if (is_null($default)) { + Console::warning("Skipping migration for Document {$document->getId()} in Collection {$collection->getId()} ({$collection->getAttribute('name')}) because of missing required attribute \"{$attribute}\" without default value."); + + continue 2; + } + $document->setAttribute($attribute, $default); + } + } + $this->populatePermissionsAttribute( + document: $document, + table: $collectionTable, + addCreatePermission: false + ); + + $this->projectDB->updateDocument($collectionTable, $document->getId(), $document); + } + $this->removeWritePermissions($collectionTable); + } + $this->removeWritePermissions($databaseTable); + + try { + $this->projectDB->deleteAttribute("database_{$database->getInternalId()}", 'permission'); + } catch (\Throwable $th) { + Console::warning("'permission' from {$databaseTable}: {$th->getMessage()}"); + } + } + } + + /** + * Removes all 'write' permissions from a table. + * + * @param string $table + * @return void + */ + protected function removeWritePermissions(string $table): void + { + try { + $this->pdo->prepare("DELETE FROM `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}_perms` WHERE _type = 'write'")->execute(); + } catch (\Throwable $th) { + Console::warning("Remove 'write' permissions from {$table}: {$th->getMessage()}"); + } + } + + /** + * Returns all columns from the Table. + * + * @param string $table + * @return array + * @throws \Exception + * @throws \PDOException + */ + protected function getSQLColumnTypes(string $table): array + { + $query = $this->pdo->prepare("SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '_{$this->project->getInternalId()}_{$table}' AND table_schema = '{$this->projectDB->getDefaultDatabase()}'"); + $query->execute(); + + return array_reduce($query->fetchAll(), function (array $carry, array $item) { + $carry[$item['COLUMN_NAME']] = $item['DATA_TYPE']; + + return $carry; + }, []); + } + + /** + * Migrates all Integer colums for timestamps to DateTime. + * + * @return void + * @throws \Exception + */ + protected function migrateDateTimeAttribute(string $table, string $attribute): void + { + $columns = $this->getSQLColumnTypes($table); + + if ($columns[$attribute] === 'int') { + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` MODIFY {$attribute} VARCHAR(64)")->execute(); + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` SET {$attribute} = IF({$attribute} = 0, NULL, FROM_UNIXTIME({$attribute}))")->execute(); + $columns[$attribute] = 'varchar'; + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + } + + if ($columns[$attribute] === 'varchar') { + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` MODIFY {$attribute} DATETIME(3)")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + } + + /** + * Skip adding filter on internal attributes. + */ + if (!str_starts_with($attribute, '_')) { + try { + /** + * Add datetime filter. + */ + $this->projectDB->updateAttributeFilters($table, ID::custom($attribute), ['datetime']); + /** + * Change data type to DateTime. + */ + $this->projectDB->updateAttribute( + collection: $table, + id: $attribute, + type: Database::VAR_DATETIME, + signed: false + ); + } catch (\Throwable $th) { + Console::warning("Add 'datetime' filter to '{$attribute}' from {$table}: {$th->getMessage()}"); + } + } + + $this->projectDB->deleteCachedCollection($table); + } + + /** + * Create the '_permissions' column to a table. + * + * @param string $table + * @return void + * @throws \Exception + * @throws \PDOException + */ + protected function createPermissionsColumn(string $table): void + { + $columns = $this->getSQLColumnTypes($table); + + if (!array_key_exists('_permissions', $columns)) { + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` ADD `_permissions` MEDIUMTEXT DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Add '_permissions' column to '{$table}': {$th->getMessage()}"); + } + } + } + + /** + * Populate '$permissions' from '$read' and '$write'. + * + * @param \Utopia\Database\Document $document + * @param null|string $table + * @param bool $addCreatePermission + * @return void + * @throws \Exception + * @throws \PDOException + */ + protected function populatePermissionsAttribute(Document &$document, ?string $table = null, bool $addCreatePermission = true): void + { + $table ??= $document->getCollection(); + + $query = $this->pdo->prepare("SELECT * FROM `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}_perms` WHERE _document = '{$document->getId()}'"); + $query->execute(); + $results = $query->fetchAll(); + $permissions = []; + + foreach ($results as $result) { + $type = $result['_type']; + $permission = $this->migratePermission($result['_permission']); + + if ($type === 'write') { + /** + * Migrate write permissions from 'role:all' to 'role:member'. + */ + if ($permission === 'role:all') { + $permission = 'role:member'; + } + + $permissions[] = "update(\"{$permission}\")"; + $permissions[] = "delete(\"{$permission}\")"; + if ($addCreatePermission) { + $permissions[] = "create(\"{$permission}\")"; + } + } else { + $permissions[] = "{$type}(\"{$permission}\")"; + } + } + + $document->setAttribute('$permissions', $permissions); + } + + /** + * Migrates a permission string + * + * @param string $permission + * @return string + */ + protected function migratePermission(string $permission): string + { + return match ($permission) { + 'role:all' => 'any', + 'role:guest' => 'guests', + 'role:member' => 'users', + default => $permission + }; + } + + /** + * Migrate all Collections. + * + * @return void + */ + protected function migrateCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + Console::log("Migrating Collection \"{$id}\""); + + $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + + switch ($id) { + case '_metadata': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + Console::log('Created new Collection "cache" collection'); + $this->createCollection('cache'); + Console::log('Created new Collection "variables" collection'); + $this->createCollection('variables'); + $this->projectDB->deleteCachedCollection($id); + break; + + case 'abuse': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + break; + + case 'attributes': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + break; + + case 'audit': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + break; + + case 'buckets': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + + try { + /** + * Create 'compression' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'compression'); + } catch (\Throwable $th) { + Console::warning("'compression' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create 'fileSecurity' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'fileSecurity'); + } catch (\Throwable $th) { + Console::warning("'fileSecurity' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_enabled' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_enabled'); + } catch (\Throwable $th) { + Console::warning("'_key_enabled' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_name' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_name'); + } catch (\Throwable $th) { + Console::warning("'_key_name' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_fileSecurity' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_fileSecurity'); + } catch (\Throwable $th) { + Console::warning("'_key_fileSecurity' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_maximumFileSize' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_maximumFileSize'); + } catch (\Throwable $th) { + Console::warning("'_key_maximumFileSize' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_encryption' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_encryption'); + } catch (\Throwable $th) { + Console::warning("'_key_encryption' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_antivirus' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_antivirus'); + } catch (\Throwable $th) { + Console::warning("'_key_antivirus' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'builds': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'startTime'); + $this->migrateDateTimeAttribute($id, 'endTime'); + break; + + case 'certificates': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'issueDate'); + $this->migrateDateTimeAttribute($id, 'renewDate'); + $this->migrateDateTimeAttribute($id, 'updated'); + break; + + case 'databases': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + break; + + case 'deployments': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + + try { + /** + * Create '_key_entrypoint' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_entrypoint'); + } catch (\Throwable $th) { + Console::warning("'_key_entrypoint' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_size' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_size'); + } catch (\Throwable $th) { + Console::warning("'_key_size' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_buildId' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_buildId'); + } catch (\Throwable $th) { + Console::warning("'_key_buildId' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_activate' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_activate'); + } catch (\Throwable $th) { + Console::warning("'_key_activate' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'domains': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'updated'); + + break; + + case 'executions': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + + try { + /** + * Create 'stdout' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'stdout'); + } catch (\Throwable $th) { + Console::warning("'stdout' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Rename 'time' to 'duration' + */ + $this->projectDB->renameAttribute($id, 'time', 'duration'); + } catch (\Throwable $th) { + Console::warning("'duration' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_trigger' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_trigger'); + } catch (\Throwable $th) { + Console::warning("'_key_trigger' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_status' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_status'); + } catch (\Throwable $th) { + Console::warning("'_key_status' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_statusCode' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_statusCode'); + } catch (\Throwable $th) { + Console::warning("'_key_statusCode' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_duration' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_duration'); + } catch (\Throwable $th) { + Console::warning("'_key_duration' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'functions': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'scheduleNext'); + $this->migrateDateTimeAttribute($id, 'schedulePrevious'); + + /** + * Migrate function variables into a new table. + */ + Console::log("Migrating Collection \"{$id}\" Variables"); + + foreach ($this->documentsIterator($id) as $function) { + $vars = $function->getAttribute('vars', []); + if (!is_array($vars)) { + continue; + } + + foreach ($vars as $key => $value) { + if ($value instanceof Document) { + continue; + } + + $variableId = ID::unique(); + $variable = new Document([ + '$id' => $variableId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'functionId' => $function->getId(), + 'functionInternalId' => $function->getInternalId(), + 'key' => (string) $key, + 'value' => (string) $value, + 'search' => implode(' ', [$variableId, $key, $function->getId()]) + ]); + $this->projectDB->createDocument('variables', $variable); + } + $this->projectDB->deleteAttribute('functions', 'vars'); + $this->createAttributeFromCollection($this->projectDB, 'functions', 'vars'); + } + try { + /** + * Create 'scheduleUpdatedAt' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'scheduleUpdatedAt'); + } catch (\Throwable $th) { + Console::warning("'scheduleUpdatedAt' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'enabled' attribute + */ + @$this->projectDB->deleteAttribute($id, 'status'); + $this->createAttributeFromCollection($this->projectDB, $id, 'enabled'); + } catch (\Throwable $th) { + Console::warning("'enabled' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create '_key_name' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_name'); + } catch (\Throwable $th) { + Console::warning("'_key_name' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_enabled' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_enabled'); + } catch (\Throwable $th) { + Console::warning("'_key_enabled' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_runtime' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_runtime'); + } catch (\Throwable $th) { + Console::warning("'_key_runtime' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_deployment' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_deployment'); + } catch (\Throwable $th) { + Console::warning("'_key_deployment' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_schedule' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_schedule'); + } catch (\Throwable $th) { + Console::warning("'_key_schedule' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_scheduleNext' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_scheduleNext'); + } catch (\Throwable $th) { + Console::warning("'_key_scheduleNext' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_schedulePrevious' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_schedulePrevious'); + } catch (\Throwable $th) { + Console::warning("'_key_schedulePrevious' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_timeout' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_timeout'); + } catch (\Throwable $th) { + Console::warning("'_key_timeout' from {$id}: {$th->getMessage()}"); + } + break; + + case 'indexes': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + + break; + + case 'keys': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'expire'); + + try { + /** + * Update 'expire' default value + */ + $this->projectDB->updateAttributeDefault('keys', 'expire', null); + } catch (\Throwable $th) { + Console::warning("'expire' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'accessedAt' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt'); + } catch (\Throwable $th) { + Console::warning("'accessedAt' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create 'sdks' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'sdks'); + } catch (\Throwable $th) { + Console::warning("'sdks' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_accessedAt' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_accessedAt'); + } catch (\Throwable $th) { + Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'memberships': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'invited'); + $this->migrateDateTimeAttribute($id, 'joined'); + + try { + /** + * Create '_key_userId' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_userId'); + } catch (\Throwable $th) { + Console::warning("'_key_userId' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_teamId' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_teamId'); + } catch (\Throwable $th) { + Console::warning("'_key_teamId' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_invited' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_invited'); + } catch (\Throwable $th) { + Console::warning("'_key_invited' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_joined' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_joined'); + } catch (\Throwable $th) { + Console::warning("'_key_joined' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_confirm' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_confirm'); + } catch (\Throwable $th) { + Console::warning("'_key_confirm' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'platforms': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + + break; + + case 'projects': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + + try { + /** + * Create '_key_name' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_name'); + } catch (\Throwable $th) { + Console::warning("'_key_name' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'realtime': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'timestamp'); + + break; + + case 'sessions': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'expire'); + $this->migrateDateTimeAttribute($id, 'providerAccessTokenExpiry'); + + break; + + case 'stats': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'time'); + + try { + /** + * Re-Create '_key_metric' index + */ + @$this->projectDB->deleteIndex($id, '_key_metric'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_period_time'); + } catch (\Throwable $th) { + Console::warning("'_key_period_time' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Re-Create '_key_metric_period' index + */ + @$this->projectDB->deleteIndex($id, '_key_metric_period'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); + } catch (\Throwable $th) { + Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'teams': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + + try { + /** + * Create '_key_name' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_name'); + } catch (\Throwable $th) { + Console::warning("'_key_name' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_total' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_total'); + } catch (\Throwable $th) { + Console::warning("'_key_total' from {$id}: {$th->getMessage()}"); + } + + break; + + case 'tokens': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'expire'); + + break; + + case 'users': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + $this->migrateDateTimeAttribute($id, 'registration'); + $this->migrateDateTimeAttribute($id, 'passwordUpdate'); + + try { + /** + * Create 'hash' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'hash'); + } catch (\Throwable $th) { + Console::warning("'hash' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create 'hashOptions' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'hashOptions'); + } catch (\Throwable $th) { + Console::warning("'hashOptions' from {$id}: {$th->getMessage()}"); + } + + /** + * Update user password before adding encrypt filter. + */ + Console::log("Migrating Collection \"{$id}\" Passwords"); + + foreach ($this->documentsIterator('users') as $user) { + /** + * Skip when no password. + */ + if (is_null($user->getAttribute('password'))) { + continue; + } + /** + * Skip when password is JSON. + */ + json_decode($user->getAttribute('password')); + if (json_last_error() === JSON_ERROR_NONE) { + continue; + } + + /** + * Add default hash. + */ + $user->setAttribute('hash', 'bcrypt'); + + /** + * Add default hash options. + */ + $user->setAttribute('hashOptions', json_encode(['cost' => 8])); + + /** + * Encrypt hashed password. + */ + $user->setAttribute('password', $this->encryptFilter($user->getAttribute('password'))); + + /** + * Migrate permissions. + */ + $this->populatePermissionsAttribute($user, addCreatePermission: false); + + $this->projectDB->updateDocument('users', $user->getId(), $user); + } + + try { + /** + * Add datetime filter to password. + */ + $this->projectDB->updateAttributeFilters($id, 'password', ['encrypt']); + } catch (\Throwable $th) { + Console::warning("Add 'encrypt' filter to 'password' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_name' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_name'); + } catch (\Throwable $th) { + Console::warning("'_key_name' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_status' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_status'); + } catch (\Throwable $th) { + Console::warning("'_key_status' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_passwordUpdate' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_passwordUpdate'); + } catch (\Throwable $th) { + Console::warning("'_key_passwordUpdate' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_registration' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_registration'); + } catch (\Throwable $th) { + Console::warning("'_key_registration' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_emailVerification' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_emailVerification'); + } catch (\Throwable $th) { + Console::warning("'_key_emailVerification' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create '_key_phoneVerification' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_phoneVerification'); + } catch (\Throwable $th) { + Console::warning("'_key_phoneVerification' from {$id}: {$th->getMessage()}"); + } + + $this->migrateStatsMetric('users.count', 'users.$all.requests.count'); + $this->migrateStatsMetric('users.create', 'users.$all.requests.create'); + $this->migrateStatsMetric('users.read', 'users.$all.requests.read'); + $this->migrateStatsMetric('users.update', 'users.$all.requests.update'); + $this->migrateStatsMetric('users.delete', 'users.$all.requests.delete'); + $this->migrateStatsMetric('users.sessions.create', 'sessions.$all.requests.create'); + $this->migrateStatsMetric('users.sessions.delete', 'sessions.$all.requests.delete'); + + foreach ($this->providers as $provider) { + $this->migrateStatsMetric("users.sessions.{$provider}.create", "sessions.$provider.requests.create"); + } + + break; + + case 'webhooks': + $this->createPermissionsColumn($id); + $this->migrateDateTimeAttribute($id, '_createdAt'); + $this->migrateDateTimeAttribute($id, '_updatedAt'); + break; + + default: + break; + } + + usleep(50000); + } + } + + /** + * Fix run on each document + * + * @param \Utopia\Database\Document $document + * @return \Utopia\Database\Document + */ + protected function fixDocument(Document $document) + { + switch ($document->getCollection()) { + case 'cache': + case 'variables': + case 'users': + /** + * skipping migration for 'cache' and 'variables'. + * 'users' already migrated. + */ + return null; + + case '_metadata': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'abuse': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'attributes': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'audit': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'buckets': + /** + * Populate permissions attribute. + * + * Note: Buckets need to migrate 'create' permissions. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'builds': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'certificates': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'databases': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'deployments': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'domains': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'executions': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'functions': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + /** + * Migrate execute permissions. + */ + $document->setAttribute('execute', array_map( + fn ($p) => $this->migratePermission($p), + $document->getAttribute('execute', []) + )); + + /** + * Set 'enabled' default. + */ + $document->setAttribute('enabled', true); + /** + * Migrate functions stats. + */ + $functionId = $document->getId(); + $this->migrateStatsMetric("functions.$functionId.executions", "executions.$functionId.compute.total"); + $this->migrateStatsMetric("functions.$functionId.failures", "executions.$functionId.compute.failure"); + $this->migrateStatsMetric("functions.$functionId.compute", "executions.$functionId.compute.time"); + + break; + + case 'indexes': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'keys': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'memberships': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'platforms': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'projects': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + /** + * Bump version number. + */ + $document->setAttribute('version', '1.0.0-RC1'); + break; + + case 'realtime': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'sessions': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'stats': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'teams': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'tokens': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'users': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + + case 'webhooks': + /** + * Populate permissions attribute. + */ + $this->populatePermissionsAttribute($document, addCreatePermission: false); + + break; + } + + return $document; + } + + protected function migrateStatsMetric(string $from, string $to): void + { + try { + $from = $this->pdo->quote($from); + $to = $this->pdo->quote($to); + + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_stats` SET metric = {$to} WHERE metric = {$from}")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating steps from {$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_stats:" . $th->getMessage()); + } + } + + /** + * Filter from the 'encrypt' filter. + * + * @param string $value + * @return string|false + */ + protected function encryptFilter(string $value): string + { + $key = App::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $tag = null; + + return json_encode([ + 'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), + 'method' => OpenSSL::CIPHER_AES_128_GCM, + 'iv' => \bin2hex($iv), + 'tag' => \bin2hex($tag ?? ''), + 'version' => '1', + ]); + } +} diff --git a/tests/unit/Event/Validator/EventValidatorTest.php b/tests/unit/Event/Validator/EventValidatorTest.php index 59a31f272d..e9f652adeb 100644 --- a/tests/unit/Event/Validator/EventValidatorTest.php +++ b/tests/unit/Event/Validator/EventValidatorTest.php @@ -4,7 +4,6 @@ namespace Tests\Unit\Event\Validator; use Appwrite\Event\Validator\Event; use PHPUnit\Framework\TestCase; -use Utopia\Config\Config; class EventValidatorTest extends TestCase { @@ -12,7 +11,6 @@ class EventValidatorTest extends TestCase public function setUp(): void { - Config::load('events', __DIR__ . '/../../../../app/config/events.php'); $this->object = new Event(); } diff --git a/tests/unit/Migration/MigrationV12Test.php b/tests/unit/Migration/MigrationV12Test.php deleted file mode 100644 index 5b88ae7f5d..0000000000 --- a/tests/unit/Migration/MigrationV12Test.php +++ /dev/null @@ -1,79 +0,0 @@ -migration = new V12(); - $reflector = new ReflectionClass('Appwrite\Migration\Version\V12'); - $this->method = $reflector->getMethod('fixDocument'); - $this->method->setAccessible(true); - } - - public function testMigrationProjects(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('project'), - '$collection' => ID::custom('projects'), - 'name' => 'Appwrite', - 'version' => '0.12.0', - 'search' => '' - ])); - - $this->assertEquals($document->getAttribute('version'), '0.13.0'); - $this->assertEquals($document->getAttribute('search'), 'project Appwrite'); - } - - public function testMigrationUsers(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('user'), - '$collection' => ID::custom('users'), - 'email' => 'test@appwrite.io', - 'name' => 'Torsten Dittmann' - ])); - - $this->assertEquals($document->getAttribute('search'), 'user test@appwrite.io Torsten Dittmann'); - } - - public function testMigrationTeams(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('team'), - '$collection' => ID::custom('teams'), - 'name' => 'Appwrite' - ])); - - $this->assertEquals($document->getAttribute('search'), 'team Appwrite'); - } - - public function testMigrationFunctions(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('function'), - '$collection' => ID::custom('functions'), - 'name' => 'My Function', - 'runtime' => 'php-8.0' - ])); - - $this->assertEquals($document->getAttribute('search'), 'function My Function php-8.0'); - } - - public function testMigrationExecutions(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('execution'), - '$collection' => ID::custom('executions'), - 'functionId' => ID::custom('function') - ])); - - $this->assertEquals($document->getAttribute('search'), 'execution function'); - } -} diff --git a/tests/unit/Migration/MigrationV13Test.php b/tests/unit/Migration/MigrationV13Test.php deleted file mode 100644 index a0daaf309d..0000000000 --- a/tests/unit/Migration/MigrationV13Test.php +++ /dev/null @@ -1,41 +0,0 @@ -migration = new V13(); - $reflector = new ReflectionClass('Appwrite\Migration\Version\V13'); - $this->method = $reflector->getMethod('fixDocument'); - $this->method->setAccessible(true); - } - - public function testMigrateFunctions(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('func'), - '$collection' => ID::custom('functions'), - 'events' => ['account.create', 'users.create'] - ])); - - $this->assertEquals($document->getAttribute('events'), ['users.*.create']); - } - - public function testMigrationWebhooks(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('webh'), - '$collection' => ID::custom('webhooks'), - 'events' => ['account.create', 'users.create'] - ])); - - $this->assertEquals($document->getAttribute('events'), ['users.*.create']); - } -} diff --git a/tests/unit/Migration/MigrationV14Test.php b/tests/unit/Migration/MigrationV14Test.php deleted file mode 100644 index 0ca249069b..0000000000 --- a/tests/unit/Migration/MigrationV14Test.php +++ /dev/null @@ -1,173 +0,0 @@ -migration = new V14(); - $reflector = new ReflectionClass('Appwrite\Migration\Version\V14'); - $this->method = $reflector->getMethod('fixDocument'); - $this->method->setAccessible(true); - } - - public function testMigrateProjects(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('projects'), - 'version' => '0.14.0' - ])); - - $this->assertEquals($document->getAttribute('version'), '0.15.0'); - $this->assertEquals($document->getAttribute('version'), '0.15.0'); - } - - public function testMigrateKeys(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => 'keys' - ])); - - $this->assertArrayHasKey('expire', $document->getArrayCopy()); - $this->assertEquals($document->getAttribute('expire'), 0); - } - - public function testMigrateWebhooks(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => 'webhooks' - ])); - - $this->assertArrayHasKey('signatureKey', $document->getArrayCopy()); - $this->assertEquals(strlen($document->getAttribute('signatureKey')), 128); - } - - public function testMigrateUsers(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('users'), - 'phoneVerification' => null - ])); - - $this->assertArrayHasKey('phoneVerification', $document->getArrayCopy()); - $this->assertFalse($document->getAttribute('phoneVerification')); - } - - public function testMigratePlatforms(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('platforms'), - '$createdAt' => null, - '$updatedAt' => null, - 'dateCreated' => 123456789, - 'dateUpdated' => 987654321 - ])); - - $this->assertEquals($document->getCreatedAt(), 123456789); - $this->assertEquals($document->getUpdatedAt(), 987654321); - } - - public function testMigrateFunctions(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('functions'), - '$createdAt' => null, - '$updatedAt' => null, - 'dateCreated' => 123456789, - 'dateUpdated' => 987654321 - ])); - - $this->assertEquals($document->getCreatedAt(), 123456789); - $this->assertEquals($document->getUpdatedAt(), 987654321); - } - - public function testMigrateDeployments(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('deployments'), - '$createdAt' => null, - 'dateCreated' => 123456789, - ])); - - $this->assertEquals($document->getCreatedAt(), 123456789); - } - - public function testMigrateExecutions(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('executions'), - '$createdAt' => null, - 'dateCreated' => 123456789, - ])); - - $this->assertEquals($document->getCreatedAt(), 123456789); - } - - public function testMigrateTeams(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('teams'), - '$createdAt' => null, - 'dateCreated' => 123456789, - ])); - - $this->assertEquals($document->getCreatedAt(), 123456789); - } - - public function testMigrateAudits(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('audit'), - 'resource' => 'collection/movies', - 'event' => 'collections.movies.create' - ])); - - $this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies'); - $this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.create'); - - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('audit'), - 'resource' => 'document/avatar', - 'event' => 'collections.movies.documents.avatar.create' - ])); - - $this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies/document/avatar'); - $this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.documents.avatar.create'); - } - - public function testMigrateStats(): void - { - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('stats'), - 'metric' => 'database.collections.62b2039844d4277495d0.documents.create' - ])); - - $this->assertEquals($document->getAttribute('metric'), 'databases.default.collections.62b2039844d4277495d0.documents.create'); - - $document = $this->fixDocument(new Document([ - '$id' => ID::custom('appwrite'), - '$collection' => ID::custom('stats'), - 'metric' => 'users.create' - ])); - - $this->assertEquals($document->getAttribute('metric'), 'users.create'); - } -}