desc('Migrations worker') ->inject('message') ->inject('dbForProject') ->inject('dbForConsole') ->callback(fn(Message $message, Database $dbForProject, Database $dbForConsole) => $this->action($message, $dbForProject, $dbForConsole)); } /** * @param Message $message * @param Database $dbForProject * @param Database $dbForConsole * @return void * @throws Exception */ public function action(Message $message, Database $dbForProject, Database $dbForConsole): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } $events = $payload['events'] ?? []; $project = new Document($payload['project'] ?? []); $migration = new Document($payload['migration'] ?? []); if ($project->getId() === 'console') { return; } $this->dbForProject = $dbForProject; $this->dbForConsole = $dbForConsole; /** * Handle Event execution. */ if (! empty($events)) { return; } $this->processMigration($project, $migration); } /** * @param string $source * @param array $credentials * @return Source * @throws Exception */ protected function processSource(string $source, array $credentials): Source { return match ($source) { Firebase::getName() => new Firebase( json_decode($credentials['serviceAccount'], true), ), Supabase::getName() => new Supabase( $credentials['endpoint'], $credentials['apiKey'], $credentials['databaseHost'], 'postgres', $credentials['username'], $credentials['password'], $credentials['port'], ), NHost::getName() => new NHost( $credentials['subdomain'], $credentials['region'], $credentials['adminSecret'], $credentials['database'], $credentials['username'], $credentials['password'], $credentials['port'], ), Appwrite::getName() => new Appwrite($credentials['projectId'], str_starts_with($credentials['endpoint'], 'http://localhost/v1') ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey']), default => throw new \Exception('Invalid source type'), }; } /** * @throws Authorization * @throws Structure * @throws Conflict * @throws \Utopia\Database\Exception * @throws Exception */ protected function updateMigrationDocument(Document $migration, Document $project): Document { /** Trigger Realtime */ $allEvents = Event::generateEvents('migrations.[migrationId].update', [ 'migrationId' => $migration->getId(), ]); $target = Realtime::fromPayload( event: $allEvents[0], payload: $migration, project: $project ); Realtime::send( projectId: 'console', payload: $migration->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'], ); Realtime::send( projectId: $project->getId(), payload: $migration->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'], ); return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); } /** * @param Document $apiKey * @return void * @throws \Utopia\Database\Exception * @throws Authorization * @throws Conflict * @throws Restricted * @throws Structure */ protected function removeAPIKey(Document $apiKey): void { $this->dbForConsole->deleteDocument('keys', $apiKey->getId()); } /** * @param Document $project * @return Document * @throws Authorization * @throws Structure * @throws \Utopia\Database\Exception * @throws Exception */ protected function generateAPIKey(Document $project): Document { $generatedSecret = bin2hex(\random_bytes(128)); $key = new Document([ '$id' => ID::unique(), '$permissions' => [ Permission::read(Role::any()), Permission::update(Role::any()), Permission::delete(Role::any()), ], 'projectInternalId' => $project->getInternalId(), 'projectId' => $project->getId(), 'name' => 'Transfer API Key', 'scopes' => [ 'users.read', 'users.write', 'teams.read', 'teams.write', 'databases.read', 'databases.write', 'collections.read', 'collections.write', 'documents.read', 'documents.write', 'buckets.read', 'buckets.write', 'files.read', 'files.write', 'functions.read', 'functions.write', ], 'expire' => null, 'sdks' => [], 'accessedAt' => null, 'secret' => $generatedSecret, ]); $this->dbForConsole->createDocument('keys', $key); $this->dbForConsole->purgeCachedDocument('projects', $project->getId()); return $key; } /** * @param Document $project * @param Document $migration * @return void * @throws Authorization * @throws Conflict * @throws Restricted * @throws Structure * @throws \Utopia\Database\Exception */ protected function processMigration(Document $project, Document $migration): void { /** * @var Document $migrationDocument * @var Transfer $transfer */ $migrationDocument = null; $transfer = null; $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); $tempAPIKey = $this->generateAPIKey($projectDocument); try { $migrationDocument = $this->dbForProject->getDocument('migrations', $migration->getId()); $migrationDocument->setAttribute('stage', 'processing'); $migrationDocument->setAttribute('status', 'processing'); $this->updateMigrationDocument($migrationDocument, $projectDocument); $source = $this->processSource($migrationDocument->getAttribute('source'), $migrationDocument->getAttribute('credentials')); $source->report(); $destination = new DestinationsAppwrite( $projectDocument->getId(), 'http://appwrite/v1', $tempAPIKey['secret'], ); $transfer = new Transfer( $source, $destination ); /** Start Transfer */ $migrationDocument->setAttribute('stage', 'migrating'); $this->updateMigrationDocument($migrationDocument, $projectDocument); $transfer->run($migrationDocument->getAttribute('resources'), function () use ($migrationDocument, $transfer, $projectDocument) { $migrationDocument->setAttribute('resourceData', json_encode($transfer->getCache())); $migrationDocument->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); $this->updateMigrationDocument($migrationDocument, $projectDocument); }); $errors = $transfer->getReport(Resource::STATUS_ERROR); if (count($errors) > 0) { $migrationDocument->setAttribute('status', 'failed'); $migrationDocument->setAttribute('stage', 'finished'); $errorMessages = []; foreach ($errors as $error) { $errorMessages[] = "Failed to transfer resource '{$error['id']}:{$error['resource']}' with message '{$error['message']}'"; } $migrationDocument->setAttribute('errors', $errorMessages); $this->updateMigrationDocument($migrationDocument, $projectDocument); return; } $migrationDocument->setAttribute('status', 'completed'); $migrationDocument->setAttribute('stage', 'finished'); } catch (\Throwable $th) { Console::error($th->getMessage()); if ($migrationDocument) { Console::error($th->getMessage()); Console::error($th->getTraceAsString()); $migrationDocument->setAttribute('status', 'failed'); $migrationDocument->setAttribute('stage', 'finished'); $migrationDocument->setAttribute('errors', [$th->getMessage()]); return; } if ($transfer) { $errors = $transfer->getReport(Resource::STATUS_ERROR); if (count($errors) > 0) { $migrationDocument->setAttribute('status', 'failed'); $migrationDocument->setAttribute('stage', 'finished'); $migrationDocument->setAttribute('errors', $errors); } } } finally { if ($migrationDocument) { $this->updateMigrationDocument($migrationDocument, $projectDocument); } if ($tempAPIKey) { $this->removeAPIKey($tempAPIKey); } } } }