diff --git a/app/cli.php b/app/cli.php index 2d12d69adb..c8ebd089b1 100644 --- a/app/cli.php +++ b/app/cli.php @@ -58,7 +58,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { ->getResource(); $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); + $dbForConsole->setNamespace('_console'); // Ensure tables exist $collections = Config::getParam('collections', [])['console']; @@ -74,7 +74,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { $pools->get('console')->reclaim(); sleep($sleep); } - } while ($attempts < $maxAttempts); + } while ($attempts < $maxAttempts && !$ready); if (!$ready) { throw new Exception("Console is not ready yet. Please try again later."); diff --git a/app/init.php b/app/init.php index 11d8c6e2cb..e5dbf183bd 100644 --- a/app/init.php +++ b/app/init.php @@ -108,7 +108,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 506; +const APP_CACHE_BUSTER = 507; const APP_VERSION_STABLE = '1.4.0'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; @@ -1116,7 +1116,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $database = new Database($dbAdapter, $cache); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; }, ['pools', 'cache']); diff --git a/app/realtime.php b/app/realtime.php index 772eee49d6..25b0532b42 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -47,7 +47,7 @@ function getConsoleDB(): Database $database = new Database($dbAdapter, getCache()); - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } diff --git a/app/worker.php b/app/worker.php index 75fc95e9ac..084569886f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -37,7 +37,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) ; $adapter = new Database($database, $cache); - $adapter->setNamespace('console'); + $adapter->setNamespace('_console'); return $adapter; }, ['cache', 'register']); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 9ea3443091..b4d09b2ca8 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -77,7 +77,11 @@ abstract class Migration Authorization::disable(); Authorization::setDefaultStatus(false); - $this->collections = array_merge([ + $this->collections = Config::getParam('collections', []); + + $projectCollections = $this->collections['projects']; + + $this->collections['projects'] = array_merge([ '_metadata' => [ '$id' => ID::custom('_metadata'), '$collection' => Database::METADATA @@ -90,7 +94,7 @@ abstract class Migration '$id' => ID::custom('abuse'), '$collection' => Database::METADATA ] - ], Config::getParam('collections', [])); + ], $projectCollections); } /** @@ -131,7 +135,14 @@ abstract class Migration */ public function forEachDocument(callable $callback): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + + $collections = match ($internalProjectId) { + 'console' => $this->collections['console'], + default => $this->collections['projects'], + }; + + foreach ($collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { continue; } @@ -148,12 +159,12 @@ abstract class Migration $old = $document->getArrayCopy(); $new = call_user_func($callback, $document); - if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) { + if (is_null($new) || $new->getArrayCopy() == $old) { return; } try { - $new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document); + $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document); } catch (\Throwable $th) { Console::error('Failed to update document: ' . $th->getMessage()); return; @@ -199,32 +210,6 @@ abstract class Migration } while (!is_null($nextDocument)); } - /** - * Checks 2 arrays for differences. - * - * @param array $array1 - * @param array $array2 - * @return bool - */ - public static function hasDifference(array $array1, array $array2): bool - { - foreach ($array1 as $key => $value) { - if (is_array($value)) { - if (!isset($array2[$key]) || !is_array($array2[$key])) { - return true; - } else { - if (self::hasDifference($value, $array2[$key])) { - return true; - } - } - } elseif (!array_key_exists($key, $array2) || $array2[$key] !== $value) { - return true; - } - } - - return false; - } - /** * Creates colletion from the config collection. * @@ -237,10 +222,15 @@ abstract class Migration { $name ??= $id; + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) { $attributes = []; $indexes = []; - $collection = $this->collections[$id]; + $collection = $this->collections[$collectionType][$id]; foreach ($collection['attributes'] as $attribute) { $attributes[] = new Document([ @@ -286,10 +276,22 @@ abstract class Migration public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$from] ?? null; - if (is_null($collection)) { - throw new Exception("Collection {$collectionId} not found"); + + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + + if ($from === 'files') { + $collectionType = 'buckets'; } + + $collection = $this->collections[$collectionType][$from] ?? null; + + if (is_null($collection)) { + throw new Exception("Collection {$from} not found"); + } + $attributes = $collection['attributes']; $attributeKey = array_search($attributeId, array_column($attributes, '$id')); @@ -332,17 +334,24 @@ abstract class Migration public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void { $from ??= $collectionId; - $collection = Config::getParam('collections', [])[$collectionId] ?? null; + + $collectionType = match ($this->project->getInternalId()) { + 'console' => 'console', + default => 'projects', + }; + + $collection = $this->collections[$collectionType][$from] ?? null; if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); } + $indexes = $collection['indexes']; $indexKey = array_search($indexId, array_column($indexes, '$id')); if ($indexKey === false) { - throw new Exception("Attribute {$indexId} not found"); + throw new Exception("Index {$indexId} not found"); } $index = $indexes[$indexKey]; diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 39522aff0b..921aacbded 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -2,12 +2,14 @@ namespace Appwrite\Migration\Version; -use Appwrite\Auth\Auth; -use Utopia\Config\Config; use Appwrite\Migration\Migration; +use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception; class V19 extends Migration { @@ -28,12 +30,6 @@ class V19 extends Migration Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); - $this->alterPermissionIndex('_metadata'); - $this->alterUidType('_metadata'); - - Console::info('Migrating Databases'); - $this->migrateDatabases(); - Console::info('Migrating Collections'); $this->migrateCollections(); @@ -42,29 +38,29 @@ class V19 extends Migration Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); + + Console::log('Cleaning Up Collections'); + $this->cleanCollections(); } /** - * Migrate all Databases. + * Migrating all Bucket tables. * * @return void * @throws \Exception + * @throws \PDOException */ - private function migrateDatabases(): void + protected function migrateBuckets(): void { - foreach ($this->documentsIterator('databases') as $database) { - Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})"); + foreach ($this->documentsIterator('buckets') as $bucket) { + $id = "bucket_{$bucket->getInternalId()}"; + Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); - $databaseTable = "database_{$database->getInternalId()}"; - - $this->alterPermissionIndex($databaseTable); - $this->alterUidType($databaseTable); - - foreach ($this->documentsIterator($databaseTable) as $collection) { - $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}"; - Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})"); - $this->alterPermissionIndex($collectionTable); - $this->alterUidType($collectionTable); + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); } } } @@ -73,36 +69,457 @@ class V19 extends Migration * Migrate all Collections. * * @return void + * @throws \Throwable + * @throws Exception */ private function migrateCollections(): void { - foreach ($this->collections as $collection) { + $internalProjectId = $this->project->getInternalId(); + $collectionType = match ($internalProjectId) { + 'console' => 'console', + default => 'projects', + }; + $collections = $this->collections[$collectionType]; + foreach ($collections as $collection) { $id = $collection['$id']; + if ($id === 'schedules' && $internalProjectId === 'console') { + continue; + } + Console::log("Migrating Collection \"{$id}\""); - $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + $this->projectDB->setNamespace("_$internalProjectId"); switch ($id) { - case 'projects': + case '_metadata': + $this->createCollection('identities'); + $this->createCollection('migrations'); + // Holding off on this until a future release + // $this->createCollection('statsLogger'); + break; + case 'attributes': + case 'indexes': try { - /** - * Create 'passwordHistory' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'smtp'); - $this->createAttributeFromCollection($this->projectDB, $id, 'templates'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->updateAttribute($id, 'databaseInternalId', required: true); } catch (\Throwable $th) { - Console::warning("'SMTP and Templates' from {$id}: {$th->getMessage()}"); + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); } + + try { + $this->projectDB->updateAttribute($id, 'collectionInternalId', required: true); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'error'); + } catch (\Throwable $th) { + Console::warning("'error' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'buckets': + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_name', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + ...$indexesToDelete + ]; + + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'builds': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'logs'); + } catch (\Throwable $th) { + Console::warning("'logs' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'certificates': + try { + $this->projectDB->renameAttribute($id, 'log', 'logs'); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'logs', size: 1000000); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'databases': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'enabled'); + } catch (\Throwable $th) { + Console::warning("'enabled' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'deployments': + $attributesToCreate = [ + 'resourceInternalId', + 'buildInternalId', + 'type', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + } catch (\Throwable $th) { + Console::warning("$attribute from {$id}: {$th->getMessage()}"); + } + } + + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_entrypoint', + '_key_resource', + '_key_resource_type', + '_key_buildId', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + '_key_resource', + '_key_resource_type', + '_key_buildId', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'executions': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId'); + } catch (\Throwable $th) { + Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); + } catch (\Throwable $th) { + Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stderr', 'errors'); + } catch (\Throwable $th) { + Console::warning("'errors' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'stdout', 'logs'); + } catch (\Throwable $th) { + Console::warning("'logs' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'statusCode', 'responseStatusCode'); + } catch (\Throwable $th) { + Console::warning("'responseStatusCode' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'files': + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_name', + '_key_signature', + '_key_mimeType', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = $indexesToDelete; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'functions': + $attributesToCreate = [ + 'live', + 'installationId', + 'installationInternalId', + 'providerRepositoryId', + 'repositoryId', + 'repositoryInternalId', + 'providerBranch', + 'providerRootDirectory', + 'providerSilentMode', + 'logging', + 'deploymentInternalId', + 'scheduleInternalId', + 'scheduleId', + 'version', + 'entrypoint', + 'commands', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + } + } + + // Recreate indexes so they're the right size + $indexesToDelete = [ + '_key_name', + '_key_runtime', + '_key_deployment', + ]; + foreach ($indexesToDelete as $index) { + try { + $this->projectDB->deleteIndex($id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $indexesToCreate = [ + ...$indexesToDelete, + '_key_installationId', + '_key_installationInternalId', + '_key_providerRepositoryId', + '_key_repositoryId', + '_key_repositoryInternalId', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'memberships': + try { + $this->projectDB->updateAttribute($id, 'teamInternalId', required: true); + } catch (\Throwable $th) { + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + // Intentional fall through to update memberships.userInternalId + case 'sessions': + case 'tokens': + try { + $this->projectDB->updateAttribute($id, 'userInternalId', required: true); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'domains': + case 'keys': + case 'platforms': + case 'webhooks': + try { + $this->projectDB->updateAttribute($id, 'projectInternalId', required: true); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'projects': + $attributesToCreate = [ + 'database', + 'smtp', + 'templates', + ]; + foreach ($attributesToCreate as $attribute) { + try { + $this->createAttributeFromCollection($this->projectDB, $id, $attribute); + } catch (\Throwable $th) { + Console::warning("'$attribute' from {$id}: {$th->getMessage()}"); + Console::warning($th->getTraceAsString()); + } + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'stats': + try { + $this->projectDB->updateAttribute($id, 'value', signed: true); + } catch (\Throwable $th) { + Console::warning("'value' from {$id}: {$th->getMessage()}"); + } + + // Holding off on these until a future release + // try { + // $this->projectDB->deleteAttribute($id, 'type'); + // $this->projectDB->deleteCachedCollection($id); + // } catch (\Throwable $th) { + // Console::warning("'type' from {$id}: {$th->getMessage()}"); + // } + + // try { + // $this->projectDB->deleteIndex($id, '_key_metric_period_time'); + // $this->projectDB->deleteCachedCollection($id); + // } catch (\Throwable $th) { + // Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + // } + + // try { + // $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); + // $this->projectDB->deleteCachedCollection($id); + // } catch (\Throwable $th) { + // Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); + // } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'users': + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'labels'); + } catch (\Throwable $th) { + Console::warning("'labels' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt'); + } catch (\Throwable $th) { + Console::warning("'accessedAt' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->updateAttribute($id, 'search', filters: ['userSearch']); + } catch (\Throwable $th) { + Console::warning("'search' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_accessedAt'); + } catch (\Throwable $th) { + Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection($id); + + break; + case 'variables': + try { + $this->projectDB->deleteIndex($id, '_key_function'); + } catch (\Throwable $th) { + Console::warning("'_key_function' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->deleteIndex($id, '_key_uniqueKey'); + } catch (\Throwable $th) { + Console::warning("'_key_uniqueKey' from {$id}: {$th->getMessage()}"); + } + + try { + $this->createAttributeFromCollection($this->projectDB, $id, 'resourceType'); + } catch (\Throwable $th) { + Console::warning("'resourceType' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionInternalId', 'resourceInternalId'); + } catch (\Throwable $th) { + Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + $this->projectDB->renameAttribute($id, 'functionId', 'resourceId'); + } catch (\Throwable $th) { + Console::warning("'resourceId' from {$id}: {$th->getMessage()}"); + } + + $indexesToCreate = [ + '_key_resourceInternalId', + '_key_resourceId_resourceType', + '_key_resourceType', + '_key_uniqueKey', + ]; + foreach ($indexesToCreate as $index) { + try { + $this->createIndexFromCollection($this->projectDB, $id, $index); + } catch (\Throwable $th) { + Console::warning("'$index' from {$id}: {$th->getMessage()}"); + } + } + + $this->projectDB->deleteCachedCollection($id); + break; default: break; } - if (!in_array($id, ['files', 'collections'])) { - $this->alterPermissionIndex($id); - $this->alterUidType($id); - } usleep(50000); } @@ -117,62 +534,145 @@ class V19 extends Migration protected function fixDocument(Document $document): Document { switch ($document->getCollection()) { + case 'attributes': + case 'indexes': + $status = $document->getAttribute('status', ''); + if ($status === 'failed') { + $document->setAttribute('error', 'Unknown problem'); + } + break; + case 'builds': + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + + $stdout = $document->getAttribute('stdout', ''); + $stderr = $document->getAttribute('stderr', ''); + $document->setAttribute('logs', $stdout . PHP_EOL . $stderr); + break; + case 'databases': + $document->setAttribute('enabled', true); + break; + case 'deployments': + $resourceId = $document->getAttribute('resourceId'); + $function = $this->projectDB->getDocument('functions', $resourceId); + $document->setAttribute('resourceInternalId', $function->getInternalId()); + + $buildId = $document->getAttribute('buildId'); + if (!empty($buildId)) { + $build = $this->projectDB->getDocument('builds', $buildId); + $document->setAttribute('buildInternalId', $build->getInternalId()); + } + + $document->setAttribute('type', 'manual'); + break; + case 'executions': + $functionId = $document->getAttribute('functionId'); + $function = $this->projectDB->getDocument('functions', $functionId); + $document->setAttribute('functionInternalId', $function->getInternalId()); + + $deploymentId = $document->getAttribute('deploymentId'); + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + break; + case 'functions': + $document->setAttribute('live', true); + $document->setAttribute('logging', true); + $document->setAttribute('version', 'v2'); + $deploymentId = $document->getAttribute('deployment'); + + if (!empty($deploymentId)) { + $deployment = $this->projectDB->getDocument('deployments', $deploymentId); + $document->setAttribute('deploymentInternalId', $deployment->getInternalId()); + $document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint')); + } + + $schedule = $this->consoleDB->createDocument('schedules', new Document([ + 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'resourceType' => 'function', + 'resourceId' => $document->getId(), + 'resourceInternalId' => $document->getInternalId(), + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $this->project->getId(), + 'schedule' => $document->getAttribute('schedule'), + 'active' => !empty($document->getAttribute('schedule')) && !empty($document->getAttribute('deployment')), + ])); + + $document->setAttribute('scheduleId', $schedule->getId()); + $document->setAttribute('scheduleInternalId', $schedule->getInternalId()); + break; case 'projects': - /** - * Bump version number. - */ $document->setAttribute('version', '1.4.0'); + $databases = Config::getParam('pools-database', []); + $database = $databases[0]; + + $document->setAttribute('database', $database); $document->setAttribute('smtp', []); $document->setAttribute('templates', []); + break; + case 'rules': + $status = 'created'; + if ($document->getAttribute('verification', false)) { + $status = 'verified'; + } + + $ruleDocument = new Document([ + 'projectId' => $this->project->getId(), + 'projectInternalId' => $this->project->getInternalId(), + 'domain' => $document->getAttribute('domain'), + 'resourceType' => 'api', + 'resourceInternalId' => '', + 'resourceId' => '', + 'status' => $status, + 'certificateId' => $document->getAttribute('certificateId'), + ]); + + try { + $this->consoleDB->createDocument('rules', $ruleDocument); + } catch (\Throwable $th) { + Console::warning("Error migrating domain {$document->getAttribute('domain')}: {$th->getMessage()}"); + } + + break; + default: break; } return $document; } - protected function alterPermissionIndex($collectionName): void + private function cleanCollections(): void { try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - DROP INDEX `_permission`, - ADD INDEX `_permission` (`_permission`, `_type`, `_document`); - ")->execute(); + $this->projectDB->deleteAttribute('projects', 'domains'); } catch (\Throwable $th) { - Console::warning($th->getMessage()); + Console::warning("'domains' from projects: {$th->getMessage()}"); } - } - protected function alterUidType($collectionName): void - { + $this->projectDB->deleteCachedCollection('projects'); + try { - $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`"; - $this->pdo->prepare(" - ALTER TABLE {$table} - CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ; - ")->execute(); + $this->projectDB->deleteAttribute('functions', 'schedule'); } catch (\Throwable $th) { - Console::warning($th->getMessage()); + Console::warning("'schedule' from functions: {$th->getMessage()}"); } - } - /** - * Migrating all Bucket tables. - * - * @return void - * @throws \Exception - * @throws \PDOException - */ - protected function migrateBuckets(): void - { - foreach ($this->documentsIterator('buckets') as $bucket) { - $id = "bucket_{$bucket->getInternalId()}"; - Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})"); - $this->alterPermissionIndex($id); - $this->alterUidType($id); + $this->projectDB->deleteCachedCollection('functions'); + + try { + $this->projectDB->deleteAttribute('builds', 'stderr'); + } catch (\Throwable $th) { + Console::warning("'stderr' from builds: {$th->getMessage()}"); } + + try { + $this->projectDB->deleteAttribute('builds', 'stdout'); + } catch (\Throwable $th) { + Console::warning("'stdout' from builds: {$th->getMessage()}"); + } + + $this->projectDB->deleteCachedCollection('builds'); } } diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 90b4234109..38e5c8aa46 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -7,10 +7,10 @@ use Utopia\CLI\Console; use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -26,20 +26,22 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') /** @TODO APP_VERSION_STABLE needs to be defined */ ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->inject('register') - ->callback(fn ($version, $register) => $this->action($version, $register)); + ->inject('cache') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); } - private function clearProjectsCache(Redis $redis, Document $project) + private function clearProjectsCache(Cache $cache, Document $project) { try { - $redis->del($redis->keys("cache-_{$project->getInternalId()}:*")); + $cache->purge("cache-_{$project->getInternalId()}:*"); } catch (\Throwable $th) { Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage()); } } - public function action(string $version, Registry $register) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -52,14 +54,6 @@ class Migrate extends Action Console::success('Starting Data Migration to version ' . $version); - $dbPool = $register->get('dbPool', true); - $redis = $register->get('cache', true); - - $cache = new Cache(new RedisCache($redis)); - - $dbForConsole = $dbPool->getDB('console', $cache); - $dbForConsole->setNamespace('_project_console'); - $console = $app->getResource('console'); $limit = 30; @@ -79,6 +73,7 @@ class Migrate extends Action } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + /** @var Migration $migration */ $migration = new $class(); while (!empty($projects)) { @@ -90,11 +85,11 @@ class Migrate extends Action continue; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); try { // TODO: Iterate through all project DBs - $projectDB = $dbPool->getDB($project->getId(), $cache); + $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) ->execute(); @@ -103,11 +98,11 @@ class Migrate extends Action throw $th; } - $this->clearProjectsCache($redis, $project); + $this->clearProjectsCache($cache, $project); } $sum = \count($projects); - $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); + $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]); $offset = $offset + $limit; $count = $count + $sum; @@ -115,7 +110,6 @@ class Migrate extends Action Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); } - Swoole\Event::wait(); // Wait for Coroutines to finish Console::success('Data Migration Completed'); } } diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 146500e743..548dc4871e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -223,7 +223,7 @@ abstract class Worker if (isset(self::$databases[$databaseName])) { $database = self::$databases[$databaseName]; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } @@ -237,7 +237,7 @@ abstract class Worker self::$databases[$databaseName] = $database; - $database->setNamespace('console'); + $database->setNamespace('_console'); return $database; } diff --git a/tests/unit/Migration/MigrationTest.php b/tests/unit/Migration/MigrationTest.php index 13f9b9a3f5..536278d55b 100644 --- a/tests/unit/Migration/MigrationTest.php +++ b/tests/unit/Migration/MigrationTest.php @@ -47,92 +47,4 @@ abstract class MigrationTest extends TestCase $this->assertArrayHasKey(APP_VERSION_STABLE, Migration::$versions); } } - - public function testHasDifference(): void - { - $this->assertFalse(Migration::hasDifference([], [])); - $this->assertFalse(Migration::hasDifference([ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ] - ], [ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ] - ])); - $this->assertFalse(Migration::hasDifference([ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ] - ], [ - 'string' => 'abc', - 'assoc' => [ - 'a' => true, - 'b' => 'abc', - 'c' => 123, - 'd' => ['a', 'b', 'c'] - ], - 'int' => 123, - 'array' => ['a', 'b', 'c'], - 'bool' => true, - - ])); - $this->assertTrue(Migration::hasDifference([ - 'a' => true - ], [ - 'b' => true - ])); - $this->assertTrue(Migration::hasDifference([ - 'a' => 'true' - ], [ - 'a' => true - ])); - $this->assertTrue(Migration::hasDifference([ - 'a' => true - ], [ - 'a' => false - ])); - $this->assertTrue(Migration::hasDifference([ - 'nested' => [ - 'a' => true - ] - ], [ - 'nested' => [] - ])); - $this->assertTrue(Migration::hasDifference([ - 'assoc' => [ - 'bool' => true, - 'string' => 'abc', - 'int' => 123, - 'array' => ['a', 'b', 'c'] - ] - ], [ - 'nested' => [ - 'a' => true, - 'int' => '123', - 'array' => ['a', 'b', 'c'] - ] - ])); - } }