From 87f79509a0634bf76206ef8e2d2722ef88176455 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 12:17:18 +0200 Subject: [PATCH 01/32] feat: migration for 0.15 collections --- src/Appwrite/Migration/Migration.php | 83 ++++++- src/Appwrite/Migration/Version/V14.php | 290 +++++++++++++++++++++++++ 2 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 src/Appwrite/Migration/Version/V14.php diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 92c5378425..0b6a34ee95 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -45,7 +45,7 @@ abstract class Migration '0.14.0' => 'V13', '0.14.1' => 'V13', '0.14.2' => 'V13', - '0.15.0' => 'V13' + '0.15.0' => 'V14' ]; /** @@ -228,6 +228,87 @@ abstract class Migration } } + /** + * Creates attribute from collections.php + * + * @param \Utopia\Database\Database $database + * @param string $collectionId + * @param string $attributeId + * @return void + * @throws \Exception + * @throws \Utopia\Database\Exception\Duplicate + * @throws \Utopia\Database\Exception\Limit + */ + public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId): void + { + $collection = Config::getParam('collections', [])[$collectionId] ?? null; + + if (is_null($collection)) { + throw new Exception("Collection {$collectionId} not found"); + } + $attributes = $collection['attributes']; + + $attributeKey = array_search($attributeId, array_column($attributes, '$id')); + + if (!$attributeKey) { + throw new Exception("Attribute {$attributeId} not found"); + } + + $attribute = $attributes[$attributeKey]; + + $database->createAttribute( + collection: $collectionId, + id: $attributeId, + type: $attribute['type'], + size: $attribute['size'], + required: $attribute['required'] ?? false, + default: $attribute['default'] ?? null, + signed: $attribute['signed'] ?? false, + array: $attribute['array'] ?? false, + format: $attribute['format'] ?? '', + formatOptions: $attribute['formatOptions'] ?? [], + filters: $attribute['filters'] ?? [], + ); + } + + /** + * Creates index from collections.php + * + * @param \Utopia\Database\Database $database + * @param string $collectionId + * @param string $indexId + * @return void + * @throws \Exception + * @throws \Utopia\Database\Exception\Duplicate + * @throws \Utopia\Database\Exception\Limit + */ + public function createIndexFromCollection(Database $database, string $collectionId, string $indexId): void + { + $collection = Config::getParam('collections', [])[$collectionId] ?? null; + + if (is_null($collection)) { + throw new Exception("Collection {$collectionId} not found"); + } + $indexes = $collection['indexes']; + + $indexKey = array_search($indexId, array_column($indexes, '$id')); + + if (!$indexKey) { + throw new Exception("Attribute {$indexId} not found"); + } + + $index = $indexes[$indexKey]; + + $database->createIndex( + collection: $collectionId, + id: $indexId, + type: $index['type'], + attributes: $index['attributes'], + lengths: $index['lengths'] ?? [], + orders: $index['orders'] ?? [] + ); + } + /** * Executes migration for set project. */ diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php new file mode 100644 index 0000000000..fb1fc20a32 --- /dev/null +++ b/src/Appwrite/Migration/Version/V14.php @@ -0,0 +1,290 @@ +project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + Console::info('Migrating Collections'); + $this->migrateCollections(); + Console::info('Migrating Documents'); + $this->forEachDocument([$this, 'fixDocument']); + } + + /** + * Migrate all Collections. + * + * @return void + */ + protected function migrateCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + Console::log("- {$id}"); + switch ($id) { + case 'attributes': + case 'indexes': + try { + /** + * Create 'collectionInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'collectionInternalId'); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Re-Create '_key_collection' index + */ + $this->projectDB->deleteIndex($id, '_key_collection'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_collection'); + } catch (\Throwable $th) { + Console::warning("'_key_collection' from {$id}: {$th->getMessage()}"); + } + + break; + case 'projects': + try { + /** + * Create 'teamInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + + break; + case 'platforms': + try { + /** + * Create 'projectInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); + } catch (\Throwable $th) { + Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_project' index + */ + $this->projectDB->deleteIndex($id, '_key_project'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'domains': + try { + /** + * Create 'projectInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_project' index + */ + $this->projectDB->deleteIndex($id, '_key_project'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'keys': + try { + /** + * Create 'projectInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'expire' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'expire'); + } catch (\Throwable $th) { + Console::warning("'expire' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_project' index + */ + $this->projectDB->deleteIndex($id, '_key_project'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'webhooks': + try { + /** + * Create 'projectInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); + } catch (\Throwable $th) { + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_project' index + */ + $this->projectDB->deleteIndex($id, '_key_project'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'users': + try { + /** + * Create 'phone' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'phone'); + } catch (\Throwable $th) { + Console::warning("'phone' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'phoneVerification' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'phoneVerification'); + } catch (\Throwable $th) { + Console::warning("'phoneVerification' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create '_key_phone' index + */ + $this->createIndexFromCollection($this->projectDB, $id, '_key_phone'); + } catch (\Throwable $th) { + Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + } + + break; + case 'tokens': + try { + /** + * Create 'userInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_user' index + */ + $this->projectDB->deleteIndex($id, '_key_user'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); + } catch (\Throwable $th) { + Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); + } + + break; + case 'sessions': + try { + /** + * Create 'userInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_user' index + */ + $this->projectDB->deleteIndex($id, '_key_user'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); + } catch (\Throwable $th) { + Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); + } + + break; + case 'memberships': + try { + /** + * Create 'teamInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Create 'userInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); + } catch (\Throwable $th) { + Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_unique' index + */ + $this->projectDB->deleteIndex($id, '_key_unique'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_unique'); + } catch (\Throwable $th) { + Console::warning("'_key_unique' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_team' index + */ + $this->projectDB->deleteIndex($id, '_key_team'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_team'); + } catch (\Throwable $th) { + Console::warning("'_key_team' from {$id}: {$th->getMessage()}"); + } + try { + /** + * Re-Create '_key_user' index + */ + $this->projectDB->deleteIndex($id, '_key_user'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); + } catch (\Throwable $th) { + Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); + } + break; + } + usleep(100000); + } + } + + /** + * Fix run on each document + * + * @param \Utopia\Database\Document $document + * @return \Utopia\Database\Document + */ + protected function fixDocument(Document $document) + { + switch ($document->getCollection()) { + case 'projects': + /** + * Bump Project version number. + */ + $document->setAttribute('version', '0.15.0'); + + break; + } + + return $document; + } +} From a8319407fc653186ed9a03c0f1281b4081266a63 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 12:17:34 +0200 Subject: [PATCH 02/32] fix: add missing indexes --- app/config/collections.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/config/collections.php b/app/config/collections.php index b8d244b6b2..6a11a3eacd 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1177,6 +1177,13 @@ $collections = [ 'lengths' => [320], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_phone', + 'type' => Database::INDEX_UNIQUE, + 'attributes' => ['phone'], + 'lengths' => [16], + 'orders' => [Database::ORDER_ASC], + ], [ '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, @@ -1744,7 +1751,7 @@ $collections = [ 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ - '$id' => '_key_internal', + '$id' => '_key_user', 'type' => Database::INDEX_KEY, 'attributes' => ['userInternalId'], 'lengths' => [Database::LENGTH_KEY], From ef6768433c15b7d47e15133391d96600ec362c73 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 12:48:56 +0200 Subject: [PATCH 03/32] feat: migrate createdAt and updatedAt to collections --- src/Appwrite/Migration/Version/V14.php | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index fb1fc20a32..b6312381a8 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -4,12 +4,21 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\Document; class V14 extends Migration { + /** + * @var \PDO $pdo + */ + private $pdo; + public function execute(): void { + global $register; + $this->pdo = $register->get('db'); + Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); Console::info('Migrating Collections'); $this->migrateCollections(); @@ -28,6 +37,18 @@ class V14 extends Migration $id = $collection['$id']; Console::log("- {$id}"); + + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD `_createdAt` int unsigned DEFAULT NULL")->execute(); + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD `_updatedAt` int unsigned DEFAULT NULL")->execute(); + $this->pdo->prepare("CREATE INDEX `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); + $this->pdo->prepare("CREATE INDEX `_updatedAt` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + usleep(100000); + switch ($id) { case 'attributes': case 'indexes': @@ -282,6 +303,9 @@ class V14 extends Migration */ $document->setAttribute('version', '0.15.0'); + break; + case '': + break; } From 718f4f1b5d453e90bb7dac32892831d034df7acb Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 15:46:05 +0200 Subject: [PATCH 04/32] fix: migration to modify tables --- src/Appwrite/Migration/Migration.php | 3 +-- src/Appwrite/Migration/Version/V14.php | 35 ++++++++++++++++++++------ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0b6a34ee95..f702bb7ac7 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -242,7 +242,6 @@ abstract class Migration public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId): void { $collection = Config::getParam('collections', [])[$collectionId] ?? null; - if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); } @@ -250,7 +249,7 @@ abstract class Migration $attributeKey = array_search($attributeId, array_column($attributes, '$id')); - if (!$attributeKey) { + if ($attributeKey === false) { throw new Exception("Attribute {$attributeId} not found"); } diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index b6312381a8..dd5dc4fa67 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -40,15 +40,39 @@ class V14 extends Migration try { $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD `_createdAt` int unsigned DEFAULT NULL")->execute(); - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD `_updatedAt` int unsigned DEFAULT NULL")->execute(); - $this->pdo->prepare("CREATE INDEX `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); - $this->pdo->prepare("CREATE INDEX `_updatedAt` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$id}_perms`")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updatedAt` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + usleep(100000); + $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + switch ($id) { case 'attributes': case 'indexes': @@ -303,9 +327,6 @@ class V14 extends Migration */ $document->setAttribute('version', '0.15.0'); - break; - case '': - break; } From 61cb9b27132259613de79673794e3095e0e440d2 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 17:59:11 +0200 Subject: [PATCH 05/32] fix: migration for internal ids on documents --- src/Appwrite/Migration/Version/V14.php | 122 +++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index dd5dc4fa67..ff278315e7 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -176,6 +176,14 @@ class V14 extends Migration break; case 'webhooks': + try { + /** + * Create 'signatureKey' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'signatureKey'); + } catch (\Throwable $th) { + Console::warning("'signatureKey' from {$id}: {$th->getMessage()}"); + } try { /** * Create 'projectInternalId' attribute @@ -327,6 +335,120 @@ class V14 extends Migration */ $document->setAttribute('version', '0.15.0'); + if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) { + $internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId(); + $document->setAttribute('teamInternalId', $internalId); + } + + break; + case 'keys': + if (is_null($document->getAttribute('expire'))) { + $document->setAttribute('expire', 0); + } + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; + case 'webhooks': + case 'domains': + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; + case 'tokens': + case 'sessions': + if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { + $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); + $document->setAttribute('userInternalId', $internalId); + } + + break; + case 'memberships': + if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { + $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); + $document->setAttribute('userInternalId', $internalId); + } + if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) { + $internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId(); + $document->setAttribute('teamInternalId', $internalId); + } + + break; + case 'attributes': + case 'indexes': + if (!empty($document->getAttribute('collectionId')) && is_null($document->getAttribute('collectionInternalId'))) { + $internalId = $this->projectDB->getDocument('collections', $document->getAttribute('collectionId'))->getInternalId(); + $document->setAttribute('collectionInternalId', $internalId); + } + + break; + case 'collections': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + if (is_null($document->getUpdateAt())) { + $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + } + + break; + case 'platforms': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + if (is_null($document->getUpdateAt())) { + $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + } + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; + case 'buckets': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + if (is_null($document->getUpdateAt())) { + $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + } + + break; + case 'files': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + + break; + case 'functions': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + if (is_null($document->getUpdateAt())) { + $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + } + + break; + case 'deployments': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + + break; + case 'executions': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + + break; + case 'teams': + if (is_null($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } + break; } From 9eb016d0f4b65acf42c89cf81d5808a7e9f431b1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 18:00:23 +0200 Subject: [PATCH 06/32] fix: realtime reconnect behaviour --- app/realtime.php | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index c14629c697..d13e1c867f 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -112,8 +112,9 @@ function getDatabase(Registry &$register, string $namespace) if (!$database->exists($database->getDefaultDatabase(), 'realtime')) { throw new Exception('Collection not ready'); } + break; // leave loop if successful - } catch (\Exception $e) { + } catch (\Throwable $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { throw new \Exception('Failed to connect to database: ' . $e->getMessage()); @@ -138,25 +139,30 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume /** * Create document for this worker to share stats across Containers. */ - go(function () use ($register, $containerId, &$statsDocument, $logError) { - try { - [$database, $returnDatabase] = getDatabase($register, '_console'); - $document = new Document([ - '$id' => $database->getId(), - '$collection' => 'realtime', - '$read' => [], - '$write' => [], - 'container' => $containerId, - 'timestamp' => time(), - 'value' => '{}' - ]); + go(function () use ($register, $containerId, &$statsDocument) { + $attempts = 0; + [$database, $returnDatabase] = getDatabase($register, '_console'); + do { + try { + $attempts++; + $document = new Document([ + '$id' => $database->getId(), + '$collection' => 'realtime', + '$read' => [], + '$write' => [], + 'container' => $containerId, + 'timestamp' => time(), + 'value' => '{}' + ]); - $statsDocument = Authorization::skip(fn () => $database->createDocument('realtime', $document)); - } catch (\Throwable $th) { - call_user_func($logError, $th, "createWorkerDocument"); - } finally { - call_user_func($returnDatabase); - } + $statsDocument = Authorization::skip(fn () => $database->createDocument('realtime', $document)); + break; + } catch (\Throwable $th) { + Console::warning("Collection not ready. Retrying connection ({$attempts})..."); + sleep(DATABASE_RECONNECT_SLEEP); + } + } while (true); + call_user_func($returnDatabase); }); /** From 67403cf95a7962157966a3936f3d31b881548d24 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 18:01:17 +0200 Subject: [PATCH 07/32] chore: update composer --- composer.json | 2 +- composer.lock | 67 +++++++++++++++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/composer.json b/composer.json index 0bb4e1b858..4bcebe3c5e 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.6.*", "utopia-php/cli": "0.12.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.18.*", + "utopia-php/database": "dev-fix-lost-transaction as 0.18.8", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index f3ef7aae6d..7db89d3de9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1593c7304ba026d4073de336227858f3", + "content-hash": "d96c8453b0715aa51cf24de92a2fa406", "packages": [ { "name": "adhocore/jwt", @@ -300,16 +300,16 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.13.0", + "version": "v1.13.1", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "afec8e58ec93d2291c127fa19709a048f28641e5" + "reference": "85df015088e00daf8ce395189de22c8eb45c8d49" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/afec8e58ec93d2291c127fa19709a048f28641e5", - "reference": "afec8e58ec93d2291c127fa19709a048f28641e5", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49", + "reference": "85df015088e00daf8ce395189de22c8eb45c8d49", "shasum": "" }, "require": { @@ -341,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.13.0" + "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1" }, - "time": "2022-04-07T14:57:22+00:00" + "time": "2022-06-20T22:56:59+00:00" }, { "name": "composer/package-versions-deprecated", @@ -481,22 +481,22 @@ }, { "name": "guzzlehttp/guzzle", - "version": "7.4.4", + "version": "7.4.5", "source": { "type": "git", "url": "https://github.com/guzzle/guzzle.git", - "reference": "e3ff079b22820c2029d4c2a87796b6a0b8716ad8" + "reference": "1dd98b0564cb3f6bd16ce683cb755f94c10fbd82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/e3ff079b22820c2029d4c2a87796b6a0b8716ad8", - "reference": "e3ff079b22820c2029d4c2a87796b6a0b8716ad8", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/1dd98b0564cb3f6bd16ce683cb755f94c10fbd82", + "reference": "1dd98b0564cb3f6bd16ce683cb755f94c10fbd82", "shasum": "" }, "require": { "ext-json": "*", "guzzlehttp/promises": "^1.5", - "guzzlehttp/psr7": "^1.8.3 || ^2.1", + "guzzlehttp/psr7": "^1.9 || ^2.4", "php": "^7.2.5 || ^8.0", "psr/http-client": "^1.0", "symfony/deprecation-contracts": "^2.2 || ^3.0" @@ -585,7 +585,7 @@ ], "support": { "issues": "https://github.com/guzzle/guzzle/issues", - "source": "https://github.com/guzzle/guzzle/tree/7.4.4" + "source": "https://github.com/guzzle/guzzle/tree/7.4.5" }, "funding": [ { @@ -601,7 +601,7 @@ "type": "tidelift" } ], - "time": "2022-06-09T21:39:15+00:00" + "time": "2022-06-20T22:16:13+00:00" }, { "name": "guzzlehttp/promises", @@ -689,16 +689,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.3.0", + "version": "2.4.0", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "83260bb50b8fc753c72d14dc1621a2dac31877ee" + "reference": "13388f00956b1503577598873fffb5ae994b5737" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/83260bb50b8fc753c72d14dc1621a2dac31877ee", - "reference": "83260bb50b8fc753c72d14dc1621a2dac31877ee", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/13388f00956b1503577598873fffb5ae994b5737", + "reference": "13388f00956b1503577598873fffb5ae994b5737", "shasum": "" }, "require": { @@ -722,7 +722,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.3-dev" + "dev-master": "2.4-dev" } }, "autoload": { @@ -784,7 +784,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.3.0" + "source": "https://github.com/guzzle/psr7/tree/2.4.0" }, "funding": [ { @@ -800,7 +800,7 @@ "type": "tidelift" } ], - "time": "2022-06-09T08:26:02+00:00" + "time": "2022-06-20T21:43:11+00:00" }, { "name": "influxdb/influxdb-php", @@ -2107,16 +2107,16 @@ }, { "name": "utopia-php/database", - "version": "0.18.2", + "version": "dev-fix-lost-transaction", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "781c31238b03ebc530a225973c4d1a921e00c2b9" + "reference": "49392d349700d65f78a6e107b953931da58fdb9d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/781c31238b03ebc530a225973c4d1a921e00c2b9", - "reference": "781c31238b03ebc530a225973c4d1a921e00c2b9", + "url": "https://api.github.com/repos/utopia-php/database/zipball/49392d349700d65f78a6e107b953931da58fdb9d", + "reference": "49392d349700d65f78a6e107b953931da58fdb9d", "shasum": "" }, "require": { @@ -2165,9 +2165,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.18.2" + "source": "https://github.com/utopia-php/database/tree/fix-lost-transaction" }, - "time": "2022-06-19T09:32:07+00:00" + "time": "2022-06-21T14:29:29+00:00" }, { "name": "utopia-php/domains", @@ -5346,9 +5346,18 @@ "time": "2022-05-17T05:48:52+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-fix-lost-transaction", + "alias": "0.18.8", + "alias_normalized": "0.18.8.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 467246cc73f392be511e2ff5bc89cd23c30788d9 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 18:32:04 +0200 Subject: [PATCH 08/32] fix: migrate webhooks and users --- src/Appwrite/Migration/Version/V14.php | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index ff278315e7..8f9e06b684 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -4,7 +4,6 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\Document; class V14 extends Migration @@ -352,8 +351,17 @@ class V14 extends Migration break; case 'webhooks': + if (empty($document->getAttribute('signatureKey'))) { + $document->setAttribute('signatureKey', \bin2hex(\random_bytes(64))); + } + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); + $document->setAttribute('projectInternalId', $internalId); + } + + break; case 'domains': - if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { + if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); $document->setAttribute('projectInternalId', $internalId); } @@ -422,6 +430,12 @@ class V14 extends Migration $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } + break; + case 'users': + if (is_null($document->getAttribute('phoneVerification'))) { + $document->setAttribute('phoneVerification', false); + } + break; case 'functions': if (is_null($document->getCreatedAt())) { From fc36e6c2d05b510de0ccc612586aa573b5fa6cec Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 19:39:05 +0200 Subject: [PATCH 09/32] fix: migration for custom collections and buckets --- app/tasks/migrate.php | 2 +- src/Appwrite/Migration/Migration.php | 2 +- src/Appwrite/Migration/Version/V14.php | 87 ++++++++++++++------------ 3 files changed, 49 insertions(+), 42 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index ff0705eb32..a7aa55aa78 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -29,7 +29,7 @@ $cli $db = $register->get('db', true); $cache = $register->get('cache', true); - + $cache->flushAll(); $cache = new Cache(new RedisCache($cache)); $projectDB = new Database(new MariaDB($db), $cache); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index f702bb7ac7..0c292d7b4c 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -292,7 +292,7 @@ abstract class Migration $indexKey = array_search($indexId, array_column($indexes, '$id')); - if (!$indexKey) { + if ($indexKey === false) { throw new Exception("Attribute {$indexId} not found"); } diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 8f9e06b684..3f0dd944f2 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -25,6 +25,42 @@ class V14 extends Migration $this->forEachDocument([$this, 'fixDocument']); } + protected function createNewMetaData(string $id): void + { + if (in_array($id, ['files'])) return; + + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$id}_perms`")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + } + /** * Migrate all Collections. * @@ -37,36 +73,7 @@ class V14 extends Migration Console::log("- {$id}"); - try { - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$id}_perms`")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updatedAt` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } + $this->createNewMetaData($id); usleep(100000); @@ -102,7 +109,7 @@ class V14 extends Migration */ $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); } catch (\Throwable $th) { - Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); } break; @@ -113,7 +120,7 @@ class V14 extends Migration */ $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); } catch (\Throwable $th) { - Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); } try { /** @@ -225,7 +232,7 @@ class V14 extends Migration */ $this->createIndexFromCollection($this->projectDB, $id, '_key_phone'); } catch (\Throwable $th) { - Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); + Console::warning("'_key_phone' from {$id}: {$th->getMessage()}"); } break; @@ -276,7 +283,7 @@ class V14 extends Migration */ $this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId'); } catch (\Throwable $th) { - Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); + Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); } try { /** @@ -388,7 +395,7 @@ class V14 extends Migration break; case 'attributes': case 'indexes': - if (!empty($document->getAttribute('collectionId')) && is_null($document->getAttribute('collectionInternalId'))) { + if (!empty($document->getAttribute('collectionId')) && is_null($document->getAttribute('collectionInternalId'))) { $internalId = $this->projectDB->getDocument('collections', $document->getAttribute('collectionId'))->getInternalId(); $document->setAttribute('collectionInternalId', $internalId); } @@ -402,6 +409,9 @@ class V14 extends Migration $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } + $internalId = $this->projectDB->getDocument('collections', $document->getId())->getInternalId(); + $this->createNewMetaData("collection_{$internalId}"); + break; case 'platforms': if (is_null($document->getCreatedAt())) { @@ -424,11 +434,8 @@ class V14 extends Migration $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } - break; - case 'files': - if (is_null($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } + $internalId = $this->projectDB->getDocument('buckets', $document->getId())->getInternalId(); + $this->createNewMetaData("bucket_{$internalId}"); break; case 'users': From 933babe9d93768995ea77a4fe841667477d1d4f3 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 19:40:25 +0200 Subject: [PATCH 10/32] chore: update composer lock --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 7db89d3de9..49f1ef45a2 100644 --- a/composer.lock +++ b/composer.lock @@ -2111,12 +2111,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "49392d349700d65f78a6e107b953931da58fdb9d" + "reference": "cca4b1862e93002de3d5543d6cb39161c4ade60c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/49392d349700d65f78a6e107b953931da58fdb9d", - "reference": "49392d349700d65f78a6e107b953931da58fdb9d", + "url": "https://api.github.com/repos/utopia-php/database/zipball/cca4b1862e93002de3d5543d6cb39161c4ade60c", + "reference": "cca4b1862e93002de3d5543d6cb39161c4ade60c", "shasum": "" }, "require": { @@ -2167,7 +2167,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/fix-lost-transaction" }, - "time": "2022-06-21T14:29:29+00:00" + "time": "2022-06-21T14:56:06+00:00" }, { "name": "utopia-php/domains", From 786abc5672daa45b91b2062dd6889da77cc2d43f Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 19:59:20 +0200 Subject: [PATCH 11/32] chore: add comments to migration --- src/Appwrite/Migration/Version/V14.php | 93 ++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 3f0dd944f2..1ef098b640 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -27,34 +27,55 @@ class V14 extends Migration protected function createNewMetaData(string $id): void { + /** + * Skip files collection. + */ if (in_array($id, ['files'])) return; try { + /** + * Replace project UID with Internal ID. + */ $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } try { + /** + * Replace project UID with Internal ID on permissions table. + */ $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$id}_perms`")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } try { + /** + * Add _createdAt attribute. + */ $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } try { + /** + * Add _updatedAt attribute. + */ $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } try { + /** + * Create index for _createdAt. + */ $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } try { + /** + * Create index for _updatedAt. + */ $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); @@ -348,9 +369,15 @@ class V14 extends Migration break; case 'keys': + /** + * Add new 'expire' attribute and default to never (0). + */ if (is_null($document->getAttribute('expire'))) { $document->setAttribute('expire', 0); } + /** + * Add Internal ID 'projectId' for Subqueries. + */ if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); $document->setAttribute('projectInternalId', $internalId); @@ -358,9 +385,15 @@ class V14 extends Migration break; case 'webhooks': + /** + * Add new 'signatureKey' attribute and generate a random value. + */ if (empty($document->getAttribute('signatureKey'))) { $document->setAttribute('signatureKey', \bin2hex(\random_bytes(64))); } + /** + * Add Internal ID 'projectId' for Subqueries. + */ if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); $document->setAttribute('projectInternalId', $internalId); @@ -368,6 +401,9 @@ class V14 extends Migration break; case 'domains': + /** + * Add Internal ID 'projectId' for Subqueries. + */ if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); $document->setAttribute('projectInternalId', $internalId); @@ -376,6 +412,9 @@ class V14 extends Migration break; case 'tokens': case 'sessions': + /** + * Add Internal ID 'userId' for Subqueries. + */ if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); $document->setAttribute('userInternalId', $internalId); @@ -383,10 +422,16 @@ class V14 extends Migration break; case 'memberships': + /** + * Add Internal ID 'userId' for Subqueries. + */ if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) { $internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId(); $document->setAttribute('userInternalId', $internalId); } + /** + * Add Internal ID 'teamId' for Subqueries. + */ if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) { $internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId(); $document->setAttribute('teamInternalId', $internalId); @@ -395,6 +440,9 @@ class V14 extends Migration break; case 'attributes': case 'indexes': + /** + * Add Internal ID 'collectionId' for Subqueries. + */ if (!empty($document->getAttribute('collectionId')) && is_null($document->getAttribute('collectionInternalId'))) { $internalId = $this->projectDB->getDocument('collections', $document->getAttribute('collectionId'))->getInternalId(); $document->setAttribute('collectionInternalId', $internalId); @@ -402,24 +450,42 @@ class V14 extends Migration break; case 'collections': + /** + * Migrate dateCreated to $createdAt. + */ if (is_null($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } + /** + * Migrate dateUpdated to $updatedAt. + */ if (is_null($document->getUpdateAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } + /** + * Migrate all Database Collections to use Internal ID. + */ $internalId = $this->projectDB->getDocument('collections', $document->getId())->getInternalId(); $this->createNewMetaData("collection_{$internalId}"); break; case 'platforms': + /** + * Migrate dateCreated to $createdAt. + */ if (is_null($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } + /** + * Migrate dateUpdated to $updatedAt. + */ if (is_null($document->getUpdateAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } + /** + * Add Internal ID 'projectId' for Subqueries. + */ if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) { $internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId(); $document->setAttribute('projectInternalId', $internalId); @@ -427,45 +493,72 @@ class V14 extends Migration break; case 'buckets': + /** + * Migrate dateCreated to $createdAt. + */ if (is_null($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } + /** + * Migrate dateUpdated to $updatedAt. + */ if (is_null($document->getUpdateAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } + /** + * Migrate all Storage Buckets to use Internal ID. + */ $internalId = $this->projectDB->getDocument('buckets', $document->getId())->getInternalId(); $this->createNewMetaData("bucket_{$internalId}"); break; case 'users': + /** + * Set 'phoneVerification' to false if not set. + */ if (is_null($document->getAttribute('phoneVerification'))) { $document->setAttribute('phoneVerification', false); } break; case 'functions': + /** + * Migrate dateCreated to $createdAt. + */ if (is_null($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } + /** + * Migrate dateUpdated to $updatedAt. + */ if (is_null($document->getUpdateAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } break; case 'deployments': + /** + * Migrate dateCreated to $createdAt. + */ if (is_null($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } break; case 'executions': + /** + * Migrate dateCreated to $createdAt. + */ if (is_null($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } break; case 'teams': + /** + * Migrate dateCreated to $createdAt. + */ if (is_null($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } From 21d4a8c14c49ebbdec825bc2932e0fd92d8d6177 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 20:05:12 +0200 Subject: [PATCH 12/32] style: fix order --- src/Appwrite/Migration/Version/V14.php | 120 +++++++++++++------------ 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 1ef098b640..6886ed5f1a 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -25,63 +25,6 @@ class V14 extends Migration $this->forEachDocument([$this, 'fixDocument']); } - protected function createNewMetaData(string $id): void - { - /** - * Skip files collection. - */ - if (in_array($id, ['files'])) return; - - try { - /** - * Replace project UID with Internal ID. - */ - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Replace project UID with Internal ID on permissions table. - */ - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$id}_perms`")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Add _createdAt attribute. - */ - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Add _updatedAt attribute. - */ - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Create index for _createdAt. - */ - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - try { - /** - * Create index for _updatedAt. - */ - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); - } catch (\Throwable $th) { - Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); - } - } - /** * Migrate all Collections. * @@ -568,4 +511,67 @@ class V14 extends Migration return $document; } + + /** + * Creates new metadata taht was introduced foir a collection and enforces the Internal ID. + * + * @param string $id + * @return void + */ + protected function createNewMetaData(string $id): void + { + /** + * Skip files collection. + */ + if (in_array($id, ['files'])) return; + + try { + /** + * Replace project UID with Internal ID. + */ + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Replace project UID with Internal ID on permissions table. + */ + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$id}_perms`")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Add _createdAt attribute. + */ + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Add _updatedAt attribute. + */ + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Create index for _createdAt. + */ + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + try { + /** + * Create index for _updatedAt. + */ + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); + } catch (\Throwable $th) { + Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); + } + } } From 12f49f66bee32b8dcc9028bc25529f0105517768 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 20:05:31 +0200 Subject: [PATCH 13/32] chore: run formatter --- src/Appwrite/Migration/Version/V14.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 6886ed5f1a..dcfca1dc04 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -523,7 +523,9 @@ class V14 extends Migration /** * Skip files collection. */ - if (in_array($id, ['files'])) return; + if (in_array($id, ['files'])) { + return; + } try { /** From 84ed87d7c304e9610aec06df6729ea0508a671b1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Jun 2022 20:21:36 +0200 Subject: [PATCH 14/32] tests: add test for migration --- tests/unit/Migration/MigrationTest.php | 1 - tests/unit/Migration/MigrationV13Test.php | 10 -- tests/unit/Migration/MigrationV14Test.php | 130 ++++++++++++++++++++++ 3 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 tests/unit/Migration/MigrationV14Test.php diff --git a/tests/unit/Migration/MigrationTest.php b/tests/unit/Migration/MigrationTest.php index 0c8c34cbc8..24aab8d1e3 100644 --- a/tests/unit/Migration/MigrationTest.php +++ b/tests/unit/Migration/MigrationTest.php @@ -4,7 +4,6 @@ namespace Appwrite\Tests; use Appwrite\Migration\Migration; use PHPUnit\Framework\TestCase; -use ReflectionClass; use ReflectionMethod; use Utopia\Database\Document; diff --git a/tests/unit/Migration/MigrationV13Test.php b/tests/unit/Migration/MigrationV13Test.php index 3d0b0e2971..ff04403536 100644 --- a/tests/unit/Migration/MigrationV13Test.php +++ b/tests/unit/Migration/MigrationV13Test.php @@ -38,14 +38,4 @@ class MigrationV13Test extends MigrationTest $this->assertEquals($document->getAttribute('events'), ['users.*.create']); } - - public function testEventsConversion() - { - $migration = new V13(); - $events = $migration->migrateEvents($migration->events); - foreach ($events as $event) { - $this->assertTrue((new Event())->isValid($event), $event); - } - $this->assertCount(44, $events); - } } diff --git a/tests/unit/Migration/MigrationV14Test.php b/tests/unit/Migration/MigrationV14Test.php new file mode 100644 index 0000000000..43b93b1539 --- /dev/null +++ b/tests/unit/Migration/MigrationV14Test.php @@ -0,0 +1,130 @@ +migration = new V14(); + $reflector = new ReflectionClass('Appwrite\Migration\Version\V14'); + $this->method = $reflector->getMethod('fixDocument'); + $this->method->setAccessible(true); + } + + public function testMigrateProject() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'projects', + 'version' => '0.14.0' + ])); + + $this->assertEquals($document->getAttribute('version'), '0.15.0'); + $this->assertEquals($document->getAttribute('version'), '0.15.0'); + } + + public function testMigrateKey() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'keys' + ])); + + $this->assertArrayHasKey('expire', $document->getArrayCopy()); + $this->assertEquals($document->getAttribute('expire'), 0); + } + + public function testMigrateWebhooks() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'webhooks' + ])); + + $this->assertArrayHasKey('signatureKey', $document->getArrayCopy()); + $this->assertEquals(strlen($document->getAttribute('signatureKey')), 128); + } + + public function testMigrateUsers() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'users', + 'phoneVerification' => null + ])); + + $this->assertArrayHasKey('phoneVerification', $document->getArrayCopy()); + $this->assertFalse($document->getAttribute('phoneVerification')); + } + + public function testMigratePlatforms() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'platforms', + '$createdAt' => null, + '$updatedAt' => null, + 'dateCreated' => 123456789, + 'dateUpdated' => 987654321 + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + $this->assertEquals($document->getUpdateAt(), 987654321); + } + + public function testMigrateFunctions() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'functions', + '$createdAt' => null, + '$updatedAt' => null, + 'dateCreated' => 123456789, + 'dateUpdated' => 987654321 + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + $this->assertEquals($document->getUpdateAt(), 987654321); + } + + public function testMigrateDeployments() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'deployments', + '$createdAt' => null, + 'dateCreated' => 123456789, + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + } + + public function testMigrateExecutions() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'executions', + '$createdAt' => null, + 'dateCreated' => 123456789, + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + } + + public function testMigrateTeams() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'teams', + '$createdAt' => null, + 'dateCreated' => 123456789, + ])); + + $this->assertEquals($document->getCreatedAt(), 123456789); + } +} From 833a7e8465a29b06e6d03034ec321225a5176d03 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 22 Jun 2022 10:56:24 +0200 Subject: [PATCH 15/32] chore: update composer --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index 4bcebe3c5e..0bb4e1b858 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.6.*", "utopia-php/cli": "0.12.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-fix-lost-transaction as 0.18.8", + "utopia-php/database": "0.18.*", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index 49f1ef45a2..fe322fb2ef 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d96c8453b0715aa51cf24de92a2fa406", + "content-hash": "1593c7304ba026d4073de336227858f3", "packages": [ { "name": "adhocore/jwt", @@ -2107,16 +2107,16 @@ }, { "name": "utopia-php/database", - "version": "dev-fix-lost-transaction", + "version": "0.18.3", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "cca4b1862e93002de3d5543d6cb39161c4ade60c" + "reference": "e7a7b9d704c1fb96611cef313a2789544c402886" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/cca4b1862e93002de3d5543d6cb39161c4ade60c", - "reference": "cca4b1862e93002de3d5543d6cb39161c4ade60c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/e7a7b9d704c1fb96611cef313a2789544c402886", + "reference": "e7a7b9d704c1fb96611cef313a2789544c402886", "shasum": "" }, "require": { @@ -2165,9 +2165,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/fix-lost-transaction" + "source": "https://github.com/utopia-php/database/tree/0.18.3" }, - "time": "2022-06-21T14:56:06+00:00" + "time": "2022-06-22T08:55:05+00:00" }, { "name": "utopia-php/domains", @@ -5346,18 +5346,9 @@ "time": "2022-05-17T05:48:52+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-fix-lost-transaction", - "alias": "0.18.8", - "alias_normalized": "0.18.8.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 604c7e499226d129910727786a6db791f215464d Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 22 Jun 2022 15:52:21 +0200 Subject: [PATCH 16/32] fix: database layer migration --- app/tasks/migrate.php | 7 +- src/Appwrite/Migration/Migration.php | 9 +- src/Appwrite/Migration/Version/V14.php | 191 +++++++++++++++++----- tests/unit/Migration/MigrationV14Test.php | 4 +- 4 files changed, 161 insertions(+), 50 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index a7aa55aa78..ff5ec3593f 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -28,9 +28,9 @@ $cli Console::success('Starting Data Migration to version ' . $version); $db = $register->get('db', true); - $cache = $register->get('cache', true); - $cache->flushAll(); - $cache = new Cache(new RedisCache($cache)); + $redis = $register->get('cache', true); + $redis->flushAll(); + $cache = new Cache(new RedisCache($redis)); $projectDB = new Database(new MariaDB($db), $cache); $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); @@ -79,5 +79,6 @@ $cli } Swoole\Event::wait(); // Wait for Coroutines to finish + $redis->flushAll(); Console::success('Data Migration Completed'); }); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0c292d7b4c..5ae4427484 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -104,7 +104,7 @@ abstract class Migration foreach ($this->collections as $collection) { if ($collection['$collection'] !== Database::METADATA) { - return; + continue; } $sum = 0; $nextDocument = null; @@ -128,7 +128,7 @@ abstract class Migration $old = $document->getArrayCopy(); $new = call_user_func($callback, $document); - if (!self::hasDifference($new->getArrayCopy(), $old)) { + if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) { return; } @@ -239,9 +239,10 @@ abstract class Migration * @throws \Utopia\Database\Exception\Duplicate * @throws \Utopia\Database\Exception\Limit */ - public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId): void + public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void { - $collection = Config::getParam('collections', [])[$collectionId] ?? null; + $from ??= $collectionId; + $collection = Config::getParam('collections', [])[$from] ?? null; if (is_null($collection)) { throw new Exception("Collection {$collectionId} not found"); } diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index dcfca1dc04..c57e985457 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -3,6 +3,7 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Document; @@ -21,10 +22,116 @@ class V14 extends Migration Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); Console::info('Migrating Collections'); $this->migrateCollections(); + Console::info('Create Default Database Layer'); + $this->createDatabaseLayer(); + if ($this->project->getId() !== 'console') { + Console::info('Migrating Database Collections'); + $this->migrateCustomCollections(); + } Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); } + public function createDatabaseLayer(): void + { + if (!$this->projectDB->exists('databases')) { + $this->createCollection('databases'); + } + + if ($this->project->getId() === 'console') { + return; + } + + try { + $this->projectDB->createDocument('databases', new Document([ + '$id' => 'default', + 'name' => 'Default', + 'search' => 'default Default' + ])); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + } + + protected function migrateCustomCollections(): void + { + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections` RENAME TO `_{$this->project->getInternalId()}_database_1`")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + try { + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections_perms` RENAME TO `_{$this->project->getInternalId()}_database_1_perms`")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + /** + * Update metadata table. + */ + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` + SET + _uid = 'database_1', + name = 'database_1' + WHERE _uid = 'collections'; + ")->execute(); + + $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseInternalId', 'collections'); + + $nextCollection = null; + + do { + $documents = $this->projectDB->find('database_1', limit: $this->limit, cursor: $nextCollection); + $count = count($documents); + + \Co\run(function (array $documents) { + foreach ($documents as $document) { + go(function (Document $collection) { + $id = $collection->getId(); + $internalId = $collection->getInternalId(); + + if ($this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), "database_1_collection_{$internalId}")) { + return; + } + Console::log("- {$id} ({$collection->getAttribute('name')})"); + + try { + /** + * Rename user's colletion table schema + */ + $this->createNewMetaData("collection_{$internalId}", "database_1_collection_{$internalId}"); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + + try { + /** + * Update metadata table. + */ + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$internalId}__metadata` + SET + _uid = 'database_1_collection_{$internalId}', + name = 'database_1_collection_{$internalId}' + WHERE _uid = 'collection_{$internalId}'; + ")->execute(); + + $collection->setAttribute('databaseInternalId', '1'); + $this->projectDB->updateDocument('database_1', $collection->getId(), $collection); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + }, $document); + } + }, $documents); + + if ($count !== $this->limit) { + $nextCollection = null; + } else { + $nextCollection = end($documents); + } + } while (!is_null($nextCollection)); + } + /** * Migrate all Collections. * @@ -46,6 +153,15 @@ class V14 extends Migration switch ($id) { case 'attributes': case 'indexes': + try { + /** + * Create 'databaseInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'databaseInternalId'); + } catch (\Throwable $th) { + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); + } + try { /** * Create 'collectionInternalId' attribute @@ -59,8 +175,8 @@ class V14 extends Migration /** * Re-Create '_key_collection' index */ - $this->projectDB->deleteIndex($id, '_key_collection'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_collection'); + @$this->projectDB->deleteIndex($id, '_key_collection'); + $this->createIndexFromCollection($this->projectDB, $id, '_key_db_collection'); } catch (\Throwable $th) { Console::warning("'_key_collection' from {$id}: {$th->getMessage()}"); } @@ -90,7 +206,7 @@ class V14 extends Migration /** * Re-Create '_key_project' index */ - $this->projectDB->deleteIndex($id, '_key_project'); + @$this->projectDB->deleteIndex($id, '_key_project'); $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); } catch (\Throwable $th) { Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); @@ -110,7 +226,7 @@ class V14 extends Migration /** * Re-Create '_key_project' index */ - $this->projectDB->deleteIndex($id, '_key_project'); + @$this->projectDB->deleteIndex($id, '_key_project'); $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); } catch (\Throwable $th) { Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); @@ -138,7 +254,7 @@ class V14 extends Migration /** * Re-Create '_key_project' index */ - $this->projectDB->deleteIndex($id, '_key_project'); + @$this->projectDB->deleteIndex($id, '_key_project'); $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); } catch (\Throwable $th) { Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); @@ -166,7 +282,7 @@ class V14 extends Migration /** * Re-Create '_key_project' index */ - $this->projectDB->deleteIndex($id, '_key_project'); + @$this->projectDB->deleteIndex($id, '_key_project'); $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); } catch (\Throwable $th) { Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); @@ -213,7 +329,7 @@ class V14 extends Migration /** * Re-Create '_key_user' index */ - $this->projectDB->deleteIndex($id, '_key_user'); + @$this->projectDB->deleteIndex($id, '_key_user'); $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); } catch (\Throwable $th) { Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); @@ -233,7 +349,7 @@ class V14 extends Migration /** * Re-Create '_key_user' index */ - $this->projectDB->deleteIndex($id, '_key_user'); + @$this->projectDB->deleteIndex($id, '_key_user'); $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); } catch (\Throwable $th) { Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); @@ -261,7 +377,7 @@ class V14 extends Migration /** * Re-Create '_key_unique' index */ - $this->projectDB->deleteIndex($id, '_key_unique'); + @$this->projectDB->deleteIndex($id, '_key_unique'); $this->createIndexFromCollection($this->projectDB, $id, '_key_unique'); } catch (\Throwable $th) { Console::warning("'_key_unique' from {$id}: {$th->getMessage()}"); @@ -270,7 +386,7 @@ class V14 extends Migration /** * Re-Create '_key_team' index */ - $this->projectDB->deleteIndex($id, '_key_team'); + @$this->projectDB->deleteIndex($id, '_key_team'); $this->createIndexFromCollection($this->projectDB, $id, '_key_team'); } catch (\Throwable $th) { Console::warning("'_key_team' from {$id}: {$th->getMessage()}"); @@ -279,7 +395,7 @@ class V14 extends Migration /** * Re-Create '_key_user' index */ - $this->projectDB->deleteIndex($id, '_key_user'); + @$this->projectDB->deleteIndex($id, '_key_user'); $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); } catch (\Throwable $th) { Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); @@ -387,31 +503,23 @@ class V14 extends Migration * Add Internal ID 'collectionId' for Subqueries. */ if (!empty($document->getAttribute('collectionId')) && is_null($document->getAttribute('collectionInternalId'))) { - $internalId = $this->projectDB->getDocument('collections', $document->getAttribute('collectionId'))->getInternalId(); + $internalId = $this->projectDB->getDocument('database_1', $document->getAttribute('collectionId'))->getInternalId(); $document->setAttribute('collectionInternalId', $internalId); } - - break; - case 'collections': /** - * Migrate dateCreated to $createdAt. + * Add Internal ID 'collectionId' for Subqueries. */ - if (is_null($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } - /** - * Migrate dateUpdated to $updatedAt. - */ - if (is_null($document->getUpdateAt())) { - $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); + if (is_null($document->getAttribute('databaseInternalId'))) { + $document->setAttribute('databaseInternalId', '1'); } - /** - * Migrate all Database Collections to use Internal ID. - */ - $internalId = $this->projectDB->getDocument('collections', $document->getId())->getInternalId(); - $this->createNewMetaData("collection_{$internalId}"); - + try { + $this->projectDB->deleteDocument($document->getCollection(), $document->getId()); + $this->projectDB->createDocument($document->getCollection(), $document->setAttribute('$id', "1_{$document->getInternalId()}_{$document->getAttribute('key')}")); + } catch (\Throwable $th) { + var_dump($th->getMessage()); + } + $document = null; break; case 'platforms': /** @@ -423,7 +531,7 @@ class V14 extends Migration /** * Migrate dateUpdated to $updatedAt. */ - if (is_null($document->getUpdateAt())) { + if (is_null($document->getUpdatedAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } /** @@ -445,7 +553,7 @@ class V14 extends Migration /** * Migrate dateUpdated to $updatedAt. */ - if (is_null($document->getUpdateAt())) { + if (is_null($document->getUpdatedAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } @@ -475,7 +583,7 @@ class V14 extends Migration /** * Migrate dateUpdated to $updatedAt. */ - if (is_null($document->getUpdateAt())) { + if (is_null($document->getUpdatedAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } @@ -518,12 +626,13 @@ class V14 extends Migration * @param string $id * @return void */ - protected function createNewMetaData(string $id): void + protected function createNewMetaData(string $id, string $to = null): void { + $to ??= $id; /** * Skip files collection. */ - if (in_array($id, ['files'])) { + if (in_array($id, ['files', 'databases'])) { return; } @@ -531,7 +640,7 @@ class V14 extends Migration /** * Replace project UID with Internal ID. */ - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$id}`")->execute(); + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$to}`")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } @@ -539,7 +648,7 @@ class V14 extends Migration /** * Replace project UID with Internal ID on permissions table. */ - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$id}_perms`")->execute(); + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$to}_perms`")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } @@ -547,7 +656,7 @@ class V14 extends Migration /** * Add _createdAt attribute. */ - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } @@ -555,7 +664,7 @@ class V14 extends Migration /** * Add _updatedAt attribute. */ - $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$id}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); + $this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } @@ -563,7 +672,7 @@ class V14 extends Migration /** * Create index for _createdAt. */ - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$id}` (`_createdAt`)")->execute(); + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$to}` (`_createdAt`)")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } @@ -571,7 +680,7 @@ class V14 extends Migration /** * Create index for _updatedAt. */ - $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$id}` (`_updatedAt`)")->execute(); + $this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$to}` (`_updatedAt`)")->execute(); } catch (\Throwable $th) { Console::warning("Migrating {$id} Collection: {$th->getMessage()}"); } diff --git a/tests/unit/Migration/MigrationV14Test.php b/tests/unit/Migration/MigrationV14Test.php index 43b93b1539..09dc7d999c 100644 --- a/tests/unit/Migration/MigrationV14Test.php +++ b/tests/unit/Migration/MigrationV14Test.php @@ -74,7 +74,7 @@ class MigrationV14Test extends MigrationTest ])); $this->assertEquals($document->getCreatedAt(), 123456789); - $this->assertEquals($document->getUpdateAt(), 987654321); + $this->assertEquals($document->getUpdatedAt(), 987654321); } public function testMigrateFunctions() @@ -89,7 +89,7 @@ class MigrationV14Test extends MigrationTest ])); $this->assertEquals($document->getCreatedAt(), 123456789); - $this->assertEquals($document->getUpdateAt(), 987654321); + $this->assertEquals($document->getUpdatedAt(), 987654321); } public function testMigrateDeployments() From a37f84da01dd1a0fdbc52fe86aee4d9372d81d67 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 22 Jun 2022 23:11:42 +0200 Subject: [PATCH 17/32] fix: migration --- app/http.php | 7 ++ src/Appwrite/Migration/Version/V14.php | 112 +++++++++++-------------- 2 files changed, 54 insertions(+), 65 deletions(-) diff --git a/app/http.php b/app/http.php index d900947dc7..72aa8183ef 100644 --- a/app/http.php +++ b/app/http.php @@ -118,6 +118,13 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { if (!$dbForConsole->getCollection($key)->isEmpty()) { continue; } + /** + * Skip to prevent 0.15 migration issues. + */ + if ($key === 'databases' && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'collections')) { + continue; + } + Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); $attributes = []; diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index c57e985457..db05f2b713 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -69,14 +69,22 @@ class V14 extends Migration /** * Update metadata table. */ - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` - SET - _uid = 'database_1', - name = 'database_1' - WHERE _uid = 'collections'; - ")->execute(); + try { + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` + SET + _uid = 'database_1', + name = 'database_1' + WHERE _uid = 'collections'; + ")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } - $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseInternalId', 'collections'); + try { + $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseInternalId', 'collections'); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } $nextCollection = null; @@ -146,8 +154,6 @@ class V14 extends Migration $this->createNewMetaData($id); - usleep(100000); - $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); switch ($id) { @@ -194,25 +200,6 @@ class V14 extends Migration break; case 'platforms': - try { - /** - * Create 'projectInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId'); - } catch (\Throwable $th) { - Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_project' index - */ - @$this->projectDB->deleteIndex($id, '_key_project'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_project'); - } catch (\Throwable $th) { - Console::warning("'_key_project' from {$id}: {$th->getMessage()}"); - } - - break; case 'domains': try { /** @@ -317,25 +304,6 @@ class V14 extends Migration break; case 'tokens': - try { - /** - * Create 'userInternalId' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId'); - } catch (\Throwable $th) { - Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Re-Create '_key_user' index - */ - @$this->projectDB->deleteIndex($id, '_key_user'); - $this->createIndexFromCollection($this->projectDB, $id, '_key_user'); - } catch (\Throwable $th) { - Console::warning("'_key_user' from {$id}: {$th->getMessage()}"); - } - - break; case 'sessions': try { /** @@ -402,7 +370,7 @@ class V14 extends Migration } break; } - usleep(100000); + usleep(50000); } } @@ -442,6 +410,33 @@ class V14 extends Migration $document->setAttribute('projectInternalId', $internalId); } + break; + case 'audit': + /** + * Add Database Layer to collection resource. + */ + if (str_starts_with($document->getAttribute('resource'), 'collection/')) { + $document + ->setAttribute('resource', "database/default/{$document->getAttribute('resource')}") + ->setAttribute('event', "databases.default.{$document->getAttribute('event')}"); + } + + if (str_starts_with($document->getAttribute('resource'), 'document/')) { + $collectionId = explode('.', $document->getAttribute('event'))[1]; + $document + ->setAttribute('resource', "database/default/collections/{$collectionId}/{$document->getAttribute('resource')}") + ->setAttribute('event', "databases.default.collections.{$collectionId}.{$document->getAttribute('event')}"); + } + + break; + case 'stats': + /** + * Add Database Layer to stats metric. + */ + if (str_starts_with($document->getAttribute('metric'), 'collections.')) { + $document->setAttribute('metric', "databases.default.{$document->getAttribute('metric')}"); + } + break; case 'webhooks': /** @@ -514,10 +509,13 @@ class V14 extends Migration } try { + /** + * Re-create Collection Document + */ $this->projectDB->deleteDocument($document->getCollection(), $document->getId()); $this->projectDB->createDocument($document->getCollection(), $document->setAttribute('$id', "1_{$document->getInternalId()}_{$document->getAttribute('key')}")); } catch (\Throwable $th) { - var_dump($th->getMessage()); + Console::warning("Create Collection Document - {$th->getMessage()}"); } $document = null; break; @@ -589,23 +587,7 @@ class V14 extends Migration break; case 'deployments': - /** - * Migrate dateCreated to $createdAt. - */ - if (is_null($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } - - break; case 'executions': - /** - * Migrate dateCreated to $createdAt. - */ - if (is_null($document->getCreatedAt())) { - $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } - - break; case 'teams': /** * Migrate dateCreated to $createdAt. From eddc33c4dbca60f29384bdb25e92b4cbb1d9d9ed Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 22 Jun 2022 23:11:52 +0200 Subject: [PATCH 18/32] fix: empty dateText filter --- public/dist/scripts/app-all.js | 2 +- public/dist/scripts/app.js | 2 +- public/scripts/filters.js | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/public/dist/scripts/app-all.js b/public/dist/scripts/app-all.js index bd8c281559..0d3ee7b0af 100644 --- a/public/dist/scripts/app-all.js +++ b/public/dist/scripts/app-all.js @@ -3803,7 +3803,7 @@ return false;};return{isRTL:isRTL,};},true);})(window);(function(window){"use st let size=element.dataset["size"]||80;let name=$value.name||$value||"";name=(typeof name!=='string')?'--':name;return def="/v1/avatars/initials?project=console"+"&name="+ encodeURIComponent(name)+"&width="+ size+"&height="+ -size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return date.format("Y-m-d",$value);}).add("dateTime",function($value,date){return date.format("Y-m-d H:i",$value);}).add("dateText",function($value,date){return date.format("d M Y",$value);}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} +size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return $value?date.format("Y-m-d",$value):"";}).add("dateTime",function($value,date){return $value?date.format("Y-m-d H:i",$value):"";}).add("dateText",function($value,date){return $value?date.format("d M Y",$value):"";}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} let value=seconds;if(seconds>=31536000){value=Math.floor(seconds/31536000);unit="year";} else if(seconds>=86400){value=Math.floor(seconds/86400);unit="day";} else if(seconds>=3600){value=Math.floor(seconds/3600);unit="hour";} diff --git a/public/dist/scripts/app.js b/public/dist/scripts/app.js index 57656cd298..7bd41521af 100644 --- a/public/dist/scripts/app.js +++ b/public/dist/scripts/app.js @@ -655,7 +655,7 @@ return false;};return{isRTL:isRTL,};},true);})(window);(function(window){"use st let size=element.dataset["size"]||80;let name=$value.name||$value||"";name=(typeof name!=='string')?'--':name;return def="/v1/avatars/initials?project=console"+"&name="+ encodeURIComponent(name)+"&width="+ size+"&height="+ -size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return date.format("Y-m-d",$value);}).add("dateTime",function($value,date){return date.format("Y-m-d H:i",$value);}).add("dateText",function($value,date){return date.format("d M Y",$value);}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} +size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return $value?date.format("Y-m-d",$value):"";}).add("dateTime",function($value,date){return $value?date.format("Y-m-d H:i",$value):"";}).add("dateText",function($value,date){return $value?date.format("d M Y",$value):"";}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";} let value=seconds;if(seconds>=31536000){value=Math.floor(seconds/31536000);unit="year";} else if(seconds>=86400){value=Math.floor(seconds/86400);unit="day";} else if(seconds>=3600){value=Math.floor(seconds/3600);unit="hour";} diff --git a/public/scripts/filters.js b/public/scripts/filters.js index bc8adfc279..eacac40c34 100644 --- a/public/scripts/filters.js +++ b/public/scripts/filters.js @@ -29,13 +29,13 @@ window.ls.filter return !Number.isNaN($value) ? $value.toLocaleString() : ""; }) .add("date", function ($value, date) { - return date.format("Y-m-d", $value); + return $value ? date.format("Y-m-d", $value) : ""; }) .add("dateTime", function ($value, date) { - return date.format("Y-m-d H:i", $value); + return $value ? date.format("Y-m-d H:i", $value) : ""; }) .add("dateText", function ($value, date) { - return date.format("d M Y", $value); + return $value ? date.format("d M Y", $value) : ""; }) .add("timeSince", function ($value) { $value = $value * 1000; From d26baf1f574c2d083ba65d8ac4079e21d0bf208b Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 23 Jun 2022 10:18:23 +0200 Subject: [PATCH 19/32] fix: migration and add more tests --- src/Appwrite/Migration/Version/V14.php | 9 +++-- tests/unit/Migration/MigrationV14Test.php | 46 ++++++++++++++++++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index db05f2b713..2098a2d8bd 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -424,8 +424,8 @@ class V14 extends Migration if (str_starts_with($document->getAttribute('resource'), 'document/')) { $collectionId = explode('.', $document->getAttribute('event'))[1]; $document - ->setAttribute('resource', "database/default/collections/{$collectionId}/{$document->getAttribute('resource')}") - ->setAttribute('event', "databases.default.collections.{$collectionId}.{$document->getAttribute('event')}"); + ->setAttribute('resource', "database/default/collection/{$collectionId}/{$document->getAttribute('resource')}") + ->setAttribute('event', "databases.default.{$document->getAttribute('event')}"); } break; @@ -433,8 +433,9 @@ class V14 extends Migration /** * Add Database Layer to stats metric. */ - if (str_starts_with($document->getAttribute('metric'), 'collections.')) { - $document->setAttribute('metric', "databases.default.{$document->getAttribute('metric')}"); + if (str_starts_with($document->getAttribute('metric'), 'database.')) { + $metric = ltrim($document->getAttribute('metric'), 'database.'); + $document->setAttribute('metric', "databases.default.{$metric}"); } break; diff --git a/tests/unit/Migration/MigrationV14Test.php b/tests/unit/Migration/MigrationV14Test.php index 09dc7d999c..ea839465af 100644 --- a/tests/unit/Migration/MigrationV14Test.php +++ b/tests/unit/Migration/MigrationV14Test.php @@ -16,7 +16,7 @@ class MigrationV14Test extends MigrationTest $this->method->setAccessible(true); } - public function testMigrateProject() + public function testMigrateProjects() { $document = $this->fixDocument(new Document([ '$id' => 'appwrite', @@ -28,7 +28,7 @@ class MigrationV14Test extends MigrationTest $this->assertEquals($document->getAttribute('version'), '0.15.0'); } - public function testMigrateKey() + public function testMigrateKeys() { $document = $this->fixDocument(new Document([ '$id' => 'appwrite', @@ -127,4 +127,46 @@ class MigrationV14Test extends MigrationTest $this->assertEquals($document->getCreatedAt(), 123456789); } + + public function testMigrateAudits() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'audit', + 'resource' => 'collection/movies', + 'event' => 'collections.movies.create' + ])); + + $this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies'); + $this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.create'); + + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'audit', + 'resource' => 'document/avatar', + 'event' => 'collections.movies.documents.avatar.create' + ])); + + $this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies/document/avatar'); + $this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.documents.avatar.create'); + } + + public function testMigrateStats() + { + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'stats', + 'metric' => 'database.collections.62b2039844d4277495d0.documents.create' + ])); + + $this->assertEquals($document->getAttribute('metric'), 'databases.default.collections.62b2039844d4277495d0.documents.create'); + + $document = $this->fixDocument(new Document([ + '$id' => 'appwrite', + '$collection' => 'stats', + 'metric' => 'users.create' + ])); + + $this->assertEquals($document->getAttribute('metric'), 'users.create'); + } } From 8e3ed52924a4d29ac52d96a90267e82e6ccc1ca1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 23 Jun 2022 10:50:53 +0200 Subject: [PATCH 20/32] style: add more comments to migration --- src/Appwrite/Migration/Version/V14.php | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 2098a2d8bd..33a56066ae 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -32,6 +32,12 @@ class V14 extends Migration $this->forEachDocument([$this, 'fixDocument']); } + /** + * Creates the default Database for existing Projects. + * + * @return void + * @throws \Throwable + */ public function createDatabaseLayer(): void { if (!$this->projectDB->exists('databases')) { @@ -53,6 +59,11 @@ class V14 extends Migration } } + /** + * Migrates all Database Collections. + * @return void + * @throws \Exception + */ protected function migrateCustomCollections(): void { try { @@ -81,6 +92,9 @@ class V14 extends Migration } try { + /** + * Add Database Internal ID for Collections. + */ $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseInternalId', 'collections'); } catch (\Throwable $th) { Console::warning($th->getMessage()); @@ -117,11 +131,11 @@ class V14 extends Migration * Update metadata table. */ $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$internalId}__metadata` - SET - _uid = 'database_1_collection_{$internalId}', - name = 'database_1_collection_{$internalId}' - WHERE _uid = 'collection_{$internalId}'; - ")->execute(); + SET + _uid = 'database_1_collection_{$internalId}', + name = 'database_1_collection_{$internalId}' + WHERE _uid = 'collection_{$internalId}'; + ")->execute(); $collection->setAttribute('databaseInternalId', '1'); $this->projectDB->updateDocument('database_1', $collection->getId(), $collection); @@ -604,7 +618,7 @@ class V14 extends Migration } /** - * Creates new metadata taht was introduced foir a collection and enforces the Internal ID. + * Creates new metadata that was introduced for a collection and enforces the Internal ID. * * @param string $id * @return void From 016d6939d575e470deae424bd8202d0624f9fcec Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 24 Jun 2022 16:04:42 +0200 Subject: [PATCH 21/32] fix: buckets --- src/Appwrite/Migration/Version/V14.php | 57 ++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 33a56066ae..c3f739bd7c 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -59,6 +59,43 @@ class V14 extends Migration } } + /** + * Migrates all Files. + * @param \Utopia\Database\Document $bucket + * @return void + * @throws \Exception + */ + protected function migrateBucketFiles(Document $bucket): void + { + $nextFile = null; + do { + $documents = $this->projectDB->find("bucket_{$bucket->getInternalId()}", limit: $this->limit, cursor: $nextFile); + $count = count($documents); + + foreach ($documents as $document) { + go(function (Document $bucket, Document $document) { + try { + /** + * Migrate $createdAt. + */ + if (empty($document->getCreatedAt())) { + $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + $this->projectDB->updateDocument("bucket_{$bucket->getInternalId()}", $document->getId(), $document); + } + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + }, $bucket, $document); + } + + if ($count !== $this->limit) { + $nextCollection = null; + } else { + $nextCollection = end($documents); + } + } while (!is_null($nextCollection)); + } + /** * Migrates all Database Collections. * @return void @@ -538,13 +575,13 @@ class V14 extends Migration /** * Migrate dateCreated to $createdAt. */ - if (is_null($document->getCreatedAt())) { + if (empty($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } /** * Migrate dateUpdated to $updatedAt. */ - if (is_null($document->getUpdatedAt())) { + if (empty($document->getUpdatedAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } /** @@ -560,13 +597,14 @@ class V14 extends Migration /** * Migrate dateCreated to $createdAt. */ - if (is_null($document->getCreatedAt())) { + if (empty($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); + } /** * Migrate dateUpdated to $updatedAt. */ - if (is_null($document->getUpdatedAt())) { + if (empty($document->getUpdatedAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } @@ -576,6 +614,11 @@ class V14 extends Migration $internalId = $this->projectDB->getDocument('buckets', $document->getId())->getInternalId(); $this->createNewMetaData("bucket_{$internalId}"); + /** + * Migrate all Storage Bucket Files. + */ + $this->migrateBucketFiles($document); + break; case 'users': /** @@ -590,13 +633,13 @@ class V14 extends Migration /** * Migrate dateCreated to $createdAt. */ - if (is_null($document->getCreatedAt())) { + if (empty($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } /** * Migrate dateUpdated to $updatedAt. */ - if (is_null($document->getUpdatedAt())) { + if (empty($document->getUpdatedAt())) { $document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated')); } @@ -607,7 +650,7 @@ class V14 extends Migration /** * Migrate dateCreated to $createdAt. */ - if (is_null($document->getCreatedAt())) { + if (empty($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); } From ababeef84d756ebef110c4ddd56d6bbdd85672da Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 24 Jun 2022 18:15:51 +0200 Subject: [PATCH 22/32] fix: update files on migration --- composer.lock | 12 ++++++------ src/Appwrite/Migration/Version/V14.php | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/composer.lock b/composer.lock index 4e93839378..0307ffae53 100644 --- a/composer.lock +++ b/composer.lock @@ -2051,16 +2051,16 @@ }, { "name": "utopia-php/database", - "version": "0.18.4", + "version": "0.18.5", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "20aa3893f2f0a970226e54c1cf7d492c44681faa" + "reference": "afdabd8fea127ec39b6e5518f7594b8f2e6c07db" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/20aa3893f2f0a970226e54c1cf7d492c44681faa", - "reference": "20aa3893f2f0a970226e54c1cf7d492c44681faa", + "url": "https://api.github.com/repos/utopia-php/database/zipball/afdabd8fea127ec39b6e5518f7594b8f2e6c07db", + "reference": "afdabd8fea127ec39b6e5518f7594b8f2e6c07db", "shasum": "" }, "require": { @@ -2109,9 +2109,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.18.4" + "source": "https://github.com/utopia-php/database/tree/0.18.5" }, - "time": "2022-06-22T09:18:07+00:00" + "time": "2022-06-24T16:14:12+00:00" }, { "name": "utopia-php/domains", diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index c3f739bd7c..cdef53d65b 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -61,9 +61,10 @@ class V14 extends Migration /** * Migrates all Files. - * @param \Utopia\Database\Document $bucket - * @return void - * @throws \Exception + * + * @param \Utopia\Database\Document $bucket + * @return void + * @throws \Exception */ protected function migrateBucketFiles(Document $bucket): void { From 2c328c744da91dd739ff5e9161b5ca8ea38176f4 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 24 Jun 2022 18:37:41 +0200 Subject: [PATCH 23/32] chore: run linter --- src/Appwrite/Migration/Version/V14.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index cdef53d65b..4cdac772ac 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -600,7 +600,6 @@ class V14 extends Migration */ if (empty($document->getCreatedAt())) { $document->setAttribute('$createdAt', $document->getAttribute('dateCreated')); - } /** * Migrate dateUpdated to $updatedAt. From 129653a6d6307559ccc72d6ebf439f7ef1863762 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 24 Jun 2022 18:46:13 +0200 Subject: [PATCH 24/32] fix: migrate databaseId --- src/Appwrite/Migration/Version/V14.php | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 4cdac772ac..160843439e 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -130,6 +130,11 @@ class V14 extends Migration } try { + /** + * Add Database ID for Collections. + */ + $this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseId', 'collections'); + /** * Add Database Internal ID for Collections. */ @@ -175,7 +180,9 @@ class V14 extends Migration WHERE _uid = 'collection_{$internalId}'; ")->execute(); - $collection->setAttribute('databaseInternalId', '1'); + $collection + ->setAttribute('databaseId', 'default') + ->setAttribute('databaseInternalId', '1'); $this->projectDB->updateDocument('database_1', $collection->getId(), $collection); } catch (\Throwable $th) { Console::warning($th->getMessage()); @@ -211,6 +218,14 @@ class V14 extends Migration switch ($id) { case 'attributes': case 'indexes': + try { + /** + * Create 'databaseInternalId' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'databaseId'); + } catch (\Throwable $th) { + Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}"); + } try { /** * Create 'databaseInternalId' attribute @@ -555,11 +570,17 @@ class V14 extends Migration $document->setAttribute('collectionInternalId', $internalId); } /** - * Add Internal ID 'collectionId' for Subqueries. + * Add Internal ID 'databaseInternalId' for Subqueries. */ if (is_null($document->getAttribute('databaseInternalId'))) { $document->setAttribute('databaseInternalId', '1'); } + /** + * Add Internal ID 'databaseInternalId' for Subqueries. + */ + if (is_null($document->getAttribute('databaseId'))) { + $document->setAttribute('databaseId', 'default'); + } try { /** From f240a11c54f0d3a78b3312e4851a39c0fab6eb41 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 13:10:04 +0200 Subject: [PATCH 25/32] fix: migrate bucket files --- src/Appwrite/Migration/Version/V14.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 160843439e..7704e462d2 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -75,6 +75,7 @@ class V14 extends Migration foreach ($documents as $document) { go(function (Document $bucket, Document $document) { + Console::log("Migrating File {$document->getId()}"); try { /** * Migrate $createdAt. @@ -90,11 +91,11 @@ class V14 extends Migration } if ($count !== $this->limit) { - $nextCollection = null; + $nextFile = null; } else { - $nextCollection = end($documents); + $nextFile = end($documents); } - } while (!is_null($nextCollection)); + } while (!is_null($nextFile)); } /** From d36e5b0167ff83f8986f9f2a69c19d185aa7d7b7 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 13:46:32 +0200 Subject: [PATCH 26/32] fix: migration speed --- src/Appwrite/Migration/Version/V14.php | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 7704e462d2..fdffc9823e 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -5,6 +5,7 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\Document; class V14 extends Migration @@ -19,6 +20,13 @@ class V14 extends Migration global $register; $this->pdo = $register->get('db'); + /** + * Disable SubQueries for Speed. + */ + foreach (['subQueryAttributes', 'subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships'] as $name) { + Database::addFilter($name, fn () => null, fn () => []); + } + Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); Console::info('Migrating Collections'); $this->migrateCollections(); @@ -40,10 +48,15 @@ class V14 extends Migration */ public function createDatabaseLayer(): void { - if (!$this->projectDB->exists('databases')) { - $this->createCollection('databases'); + try { + if (!$this->projectDB->exists('databases')) { + $this->createCollection('databases'); + } + } catch (\Throwable $th) { + Console::warning($th->getMessage()); } + if ($this->project->getId() === 'console') { return; } @@ -649,7 +662,7 @@ class V14 extends Migration if (is_null($document->getAttribute('phoneVerification'))) { $document->setAttribute('phoneVerification', false); } - + var_dump($document->getArrayCopy()); break; case 'functions': /** From d945cb2a721e70203f0677cc260cd50b75a48da9 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 13:58:21 +0200 Subject: [PATCH 27/32] fix: remove var dump --- src/Appwrite/Migration/Version/V14.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index fdffc9823e..3956423b5e 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -662,7 +662,7 @@ class V14 extends Migration if (is_null($document->getAttribute('phoneVerification'))) { $document->setAttribute('phoneVerification', false); } - var_dump($document->getArrayCopy()); + break; case 'functions': /** From 30557bbe4e939d61f594e9d8ccee6b8757521aaf Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 15:57:24 +0200 Subject: [PATCH 28/32] fix: docker compose in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3b6524d159..a863f93eb9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -244,7 +244,7 @@ RUN \ RUN \ mkdir -p $DOCKER_CONFIG/cli-plugins \ - && ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then $ARCH="armv7"; fi \ + && ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then ARCH="armv7"; fi \ && curl -SL https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-linux-$ARCH -o $DOCKER_CONFIG/cli-plugins/docker-compose \ && chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose From c3c6d00a9cbea9d66d559bb0017362a98d8453a0 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 16:58:57 +0200 Subject: [PATCH 29/32] fix: case for console projects --- src/Appwrite/Migration/Migration.php | 3 +++ src/Appwrite/Migration/Version/V14.php | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 5ae4427484..69f3b856e2 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -106,6 +106,9 @@ abstract class Migration if ($collection['$collection'] !== Database::METADATA) { continue; } + if ($collection['$id'] === 'databases') { + continue; + } $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collection['$id']); diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index 3956423b5e..ff68859418 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -20,6 +20,10 @@ class V14 extends Migration global $register; $this->pdo = $register->get('db'); + if ($this->project->getId() === 'console' && $this->project->getInternalId() !== 'console') { + return; + } + /** * Disable SubQueries for Speed. */ @@ -56,8 +60,7 @@ class V14 extends Migration Console::warning($th->getMessage()); } - - if ($this->project->getId() === 'console') { + if ($this->project->getInternalId() === 'console') { return; } From c0649f73857bec123d5e960fb04f51a3d4ecf5f6 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 18:22:29 +0200 Subject: [PATCH 30/32] fix: migration --- src/Appwrite/Migration/Migration.php | 4 +--- src/Appwrite/Migration/Version/V14.php | 7 ++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 69f3b856e2..a133af08ac 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -106,9 +106,7 @@ abstract class Migration if ($collection['$collection'] !== Database::METADATA) { continue; } - if ($collection['$id'] === 'databases') { - continue; - } + $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collection['$id']); diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index ff68859418..a7627d859c 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -172,9 +172,6 @@ class V14 extends Migration $id = $collection->getId(); $internalId = $collection->getInternalId(); - if ($this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), "database_1_collection_{$internalId}")) { - return; - } Console::log("- {$id} ({$collection->getAttribute('name')})"); try { @@ -196,7 +193,11 @@ class V14 extends Migration name = 'database_1_collection_{$internalId}' WHERE _uid = 'collection_{$internalId}'; ")->execute(); + } catch (\Throwable $th) { + Console::warning($th->getMessage()); + } + try { $collection ->setAttribute('databaseId', 'default') ->setAttribute('databaseInternalId', '1'); From ff219cf91ad3948353655757471953dda6b57593 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 18:26:24 +0200 Subject: [PATCH 31/32] fix: project internal id migration --- src/Appwrite/Migration/Version/V14.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index a7627d859c..f8edf4d890 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -187,7 +187,7 @@ class V14 extends Migration /** * Update metadata table. */ - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$internalId}__metadata` + $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` SET _uid = 'database_1_collection_{$internalId}', name = 'database_1_collection_{$internalId}' From def1c3537777c30879ee3059c8526adae1704976 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Jun 2022 18:44:39 +0200 Subject: [PATCH 32/32] chore: add more comments --- src/Appwrite/Migration/Version/V14.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Appwrite/Migration/Version/V14.php b/src/Appwrite/Migration/Version/V14.php index f8edf4d890..308850d2c9 100644 --- a/src/Appwrite/Migration/Version/V14.php +++ b/src/Appwrite/Migration/Version/V14.php @@ -198,6 +198,9 @@ class V14 extends Migration } try { + /** + * Update internal ID's. + */ $collection ->setAttribute('databaseId', 'default') ->setAttribute('databaseInternalId', '1');