Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-24 09:28:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #3414 from appwrite/feat-migration-0-15-x

Browse source 
feat: migration for 0.15.x
This commit is contained in:
Torsten Dittmann 2022-06-27 18:56:18 +02:00 committed by GitHub
commit 4038abbe10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 1051 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -244,7 +244,7 @@ RUN \
RUN \
mkdir -p $DOCKER_CONFIG/cli-plugins \
&& ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then $ARCH="armv7"; fi \
&& ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then ARCH="armv7"; fi \
&& curl -SL https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-linux-$ARCH -o $DOCKER_CONFIG/cli-plugins/docker-compose \
&& chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose

7
app/http.php
View file

@ -118,6 +118,13 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
if (!$dbForConsole->getCollection($key)->isEmpty()) {
continue;
}
/**
* Skip to prevent 0.15 migration issues.
*/
if ($key === 'databases' && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'collections')) {
continue;
}
Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...');
$attributes = [];

1
app/realtime.php
View file

@ -112,6 +112,7 @@ function getDatabase(Registry &$register, string $namespace)
if (!$database->exists($database->getDefaultDatabase(), 'realtime')) {
throw new Exception('Collection not ready');
}
break; // leave loop if successful
} catch (\Throwable $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");

7
app/tasks/migrate.php
View file

@ -28,9 +28,9 @@ $cli
Console::success('Starting Data Migration to version ' . $version);
$db = $register->get('db', true);
$cache = $register->get('cache', true);
$cache = new Cache(new RedisCache($cache));
$redis = $register->get('cache', true);
$redis->flushAll();
$cache = new Cache(new RedisCache($redis));
$projectDB = new Database(new MariaDB($db), $cache);
$projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
@ -79,5 +79,6 @@ $cli
}
Swoole\Event::wait(); // Wait for Coroutines to finish
$redis->flushAll();
Console::success('Data Migration Completed');
});

12
composer.lock generated
View file

@ -2051,16 +2051,16 @@
},
{
"name": "utopia-php/database",
"version": "0.18.4",
"version": "0.18.5",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "20aa3893f2f0a970226e54c1cf7d492c44681faa"
"reference": "afdabd8fea127ec39b6e5518f7594b8f2e6c07db"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/20aa3893f2f0a970226e54c1cf7d492c44681faa",
"reference": "20aa3893f2f0a970226e54c1cf7d492c44681faa",
"url": "https://api.github.com/repos/utopia-php/database/zipball/afdabd8fea127ec39b6e5518f7594b8f2e6c07db",
"reference": "afdabd8fea127ec39b6e5518f7594b8f2e6c07db",
"shasum": ""
},
"require": {
@ -2109,9 +2109,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.18.4"
"source": "https://github.com/utopia-php/database/tree/0.18.5"
},
"time": "2022-06-22T09:18:07+00:00"
"time": "2022-06-24T16:14:12+00:00"
},
{
"name": "utopia-php/domains",

2
public/dist/scripts/app-all.js vendored
View file

@ -3803,7 +3803,7 @@ return false;};return{isRTL:isRTL,};},true);})(window);(function(window){"use st
let size=element.dataset["size"]||80;let name=$value.name||$value||"";name=(typeof name!=='string')?'--':name;return def="/v1/avatars/initials?project=console"+"&name="+
encodeURIComponent(name)+"&width="+
size+"&height="+
size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return date.format("Y-m-d",$value);}).add("dateTime",function($value,date){return date.format("Y-m-d H:i",$value);}).add("dateText",function($value,date){return date.format("d M Y",$value);}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";}
size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return $value?date.format("Y-m-d",$value):"";}).add("dateTime",function($value,date){return $value?date.format("Y-m-d H:i",$value):"";}).add("dateText",function($value,date){return $value?date.format("d M Y",$value):"";}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";}
let value=seconds;if(seconds>=31536000){value=Math.floor(seconds/31536000);unit="year";}
else if(seconds>=86400){value=Math.floor(seconds/86400);unit="day";}
else if(seconds>=3600){value=Math.floor(seconds/3600);unit="hour";}

2
public/dist/scripts/app.js vendored
View file

@ -655,7 +655,7 @@ return false;};return{isRTL:isRTL,};},true);})(window);(function(window){"use st
let size=element.dataset["size"]||80;let name=$value.name||$value||"";name=(typeof name!=='string')?'--':name;return def="/v1/avatars/initials?project=console"+"&name="+
encodeURIComponent(name)+"&width="+
size+"&height="+
size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return date.format("Y-m-d",$value);}).add("dateTime",function($value,date){return date.format("Y-m-d H:i",$value);}).add("dateText",function($value,date){return date.format("d M Y",$value);}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";}
size;}).add("selectedCollection",function($value,router){return $value===router.params.collectionId?"selected":"";}).add("selectedDocument",function($value,router){return $value===router.params.documentId?"selected":"";}).add("localeString",function($value){$value=parseInt($value);return!Number.isNaN($value)?$value.toLocaleString():"";}).add("date",function($value,date){return $value?date.format("Y-m-d",$value):"";}).add("dateTime",function($value,date){return $value?date.format("Y-m-d H:i",$value):"";}).add("dateText",function($value,date){return $value?date.format("d M Y",$value):"";}).add("timeSince",function($value){$value=$value*1000;let seconds=Math.floor((Date.now()-$value)/1000);let unit="second";let direction="ago";if(seconds<0){seconds=-seconds;direction="from now";}
let value=seconds;if(seconds>=31536000){value=Math.floor(seconds/31536000);unit="year";}
else if(seconds>=86400){value=Math.floor(seconds/86400);unit="day";}
else if(seconds>=3600){value=Math.floor(seconds/3600);unit="hour";}

6
public/scripts/filters.js
View file

@ -29,13 +29,13 @@ window.ls.filter
return !Number.isNaN($value) ? $value.toLocaleString() : "";
})
.add("date", function ($value, date) {
return date.format("Y-m-d", $value);
return $value ? date.format("Y-m-d", $value) : "";
})
.add("dateTime", function ($value, date) {
return date.format("Y-m-d H:i", $value);
return $value ? date.format("Y-m-d H:i", $value) : "";
})
.add("dateText", function ($value, date) {
return date.format("d M Y", $value);
return $value ? date.format("d M Y", $value) : "";
})
.add("timeSince", function ($value) {
$value = $value * 1000;

88
src/Appwrite/Migration/Migration.php
View file

@ -45,7 +45,7 @@ abstract class Migration
'0.14.0' => 'V13',
'0.14.1' => 'V13',
'0.14.2' => 'V13',
'0.15.0' => 'V13'
'0.15.0' => 'V14'
];
/**
@ -104,8 +104,9 @@ abstract class Migration
foreach ($this->collections as $collection) {
if ($collection['$collection'] !== Database::METADATA) {
return;
continue;
}
$sum = 0;
$nextDocument = null;
$collectionCount = $this->projectDB->count($collection['$id']);
@ -128,7 +129,7 @@ abstract class Migration
$old = $document->getArrayCopy();
$new = call_user_func($callback, $document);
if (!self::hasDifference($new->getArrayCopy(), $old)) {
if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) {
return;
}
@ -228,6 +229,87 @@ abstract class Migration
}
}
/**
* Creates attribute from collections.php
*
* @param \Utopia\Database\Database $database
* @param string $collectionId
* @param string $attributeId
* @return void
* @throws \Exception
* @throws \Utopia\Database\Exception\Duplicate
* @throws \Utopia\Database\Exception\Limit
*/
public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void
{
$from ??= $collectionId;
$collection = Config::getParam('collections', [])[$from] ?? null;
if (is_null($collection)) {
throw new Exception("Collection {$collectionId} not found");
}
$attributes = $collection['attributes'];
$attributeKey = array_search($attributeId, array_column($attributes, '$id'));
if ($attributeKey === false) {
throw new Exception("Attribute {$attributeId} not found");
}
$attribute = $attributes[$attributeKey];
$database->createAttribute(
collection: $collectionId,
id: $attributeId,
type: $attribute['type'],
size: $attribute['size'],
required: $attribute['required'] ?? false,
default: $attribute['default'] ?? null,
signed: $attribute['signed'] ?? false,
array: $attribute['array'] ?? false,
format: $attribute['format'] ?? '',
formatOptions: $attribute['formatOptions'] ?? [],
filters: $attribute['filters'] ?? [],
);
}
/**
* Creates index from collections.php
*
* @param \Utopia\Database\Database $database
* @param string $collectionId
* @param string $indexId
* @return void
* @throws \Exception
* @throws \Utopia\Database\Exception\Duplicate
* @throws \Utopia\Database\Exception\Limit
*/
public function createIndexFromCollection(Database $database, string $collectionId, string $indexId): void
{
$collection = Config::getParam('collections', [])[$collectionId] ?? null;
if (is_null($collection)) {
throw new Exception("Collection {$collectionId} not found");
}
$indexes = $collection['indexes'];
$indexKey = array_search($indexId, array_column($indexes, '$id'));
if ($indexKey === false) {
throw new Exception("Attribute {$indexId} not found");
}
$index = $indexes[$indexKey];
$database->createIndex(
collection: $collectionId,
id: $indexId,
type: $index['type'],
attributes: $index['attributes'],
lengths: $index['lengths'] ?? [],
orders: $index['orders'] ?? []
);
}
/**
* Executes migration for set project.
*/

770
src/Appwrite/Migration/Version/V14.php Normal file
View file

@ -0,0 +1,770 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
class V14 extends Migration
{
/**
* @var \PDO $pdo
*/
private $pdo;
public function execute(): void
{
global $register;
$this->pdo = $register->get('db');
if ($this->project->getId() === 'console' && $this->project->getInternalId() !== 'console') {
return;
}
/**
* Disable SubQueries for Speed.
*/
foreach (['subQueryAttributes', 'subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships'] as $name) {
Database::addFilter($name, fn () => null, fn () => []);
}
Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
Console::info('Migrating Collections');
$this->migrateCollections();
Console::info('Create Default Database Layer');
$this->createDatabaseLayer();
if ($this->project->getId() !== 'console') {
Console::info('Migrating Database Collections');
$this->migrateCustomCollections();
}
Console::info('Migrating Documents');
$this->forEachDocument([$this, 'fixDocument']);
}
/**
* Creates the default Database for existing Projects.
*
* @return void
* @throws \Throwable
*/
public function createDatabaseLayer(): void
{
try {
if (!$this->projectDB->exists('databases')) {
$this->createCollection('databases');
}
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
if ($this->project->getInternalId() === 'console') {
return;
}
try {
$this->projectDB->createDocument('databases', new Document([
'$id' => 'default',
'name' => 'Default',
'search' => 'default Default'
]));
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
}
/**
* Migrates all Files.
*
* @param \Utopia\Database\Document $bucket
* @return void
* @throws \Exception
*/
protected function migrateBucketFiles(Document $bucket): void
{
$nextFile = null;
do {
$documents = $this->projectDB->find("bucket_{$bucket->getInternalId()}", limit: $this->limit, cursor: $nextFile);
$count = count($documents);
foreach ($documents as $document) {
go(function (Document $bucket, Document $document) {
Console::log("Migrating File {$document->getId()}");
try {
/**
* Migrate $createdAt.
*/
if (empty($document->getCreatedAt())) {
$document->setAttribute('$createdAt', $document->getAttribute('dateCreated'));
$this->projectDB->updateDocument("bucket_{$bucket->getInternalId()}", $document->getId(), $document);
}
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
}, $bucket, $document);
}
if ($count !== $this->limit) {
$nextFile = null;
} else {
$nextFile = end($documents);
}
} while (!is_null($nextFile));
}
/**
* Migrates all Database Collections.
* @return void
* @throws \Exception
*/
protected function migrateCustomCollections(): void
{
try {
$this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections` RENAME TO `_{$this->project->getInternalId()}_database_1`")->execute();
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
try {
$this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_collections_perms` RENAME TO `_{$this->project->getInternalId()}_database_1_perms`")->execute();
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
/**
* Update metadata table.
*/
try {
$this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata`
SET
_uid = 'database_1',
name = 'database_1'
WHERE _uid = 'collections';
")->execute();
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
try {
/**
* Add Database ID for Collections.
*/
$this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseId', 'collections');
/**
* Add Database Internal ID for Collections.
*/
$this->createAttributeFromCollection($this->projectDB, 'database_1', 'databaseInternalId', 'collections');
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
$nextCollection = null;
do {
$documents = $this->projectDB->find('database_1', limit: $this->limit, cursor: $nextCollection);
$count = count($documents);
\Co\run(function (array $documents) {
foreach ($documents as $document) {
go(function (Document $collection) {
$id = $collection->getId();
$internalId = $collection->getInternalId();
Console::log("- {$id} ({$collection->getAttribute('name')})");
try {
/**
* Rename user's colletion table schema
*/
$this->createNewMetaData("collection_{$internalId}", "database_1_collection_{$internalId}");
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
try {
/**
* Update metadata table.
*/
$this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata`
SET
_uid = 'database_1_collection_{$internalId}',
name = 'database_1_collection_{$internalId}'
WHERE _uid = 'collection_{$internalId}';
")->execute();
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
try {
/**
* Update internal ID's.
*/
$collection
->setAttribute('databaseId', 'default')
->setAttribute('databaseInternalId', '1');
$this->projectDB->updateDocument('database_1', $collection->getId(), $collection);
} catch (\Throwable $th) {
Console::warning($th->getMessage());
}
}, $document);
}
}, $documents);
if ($count !== $this->limit) {
$nextCollection = null;
} else {
$nextCollection = end($documents);
}
} while (!is_null($nextCollection));
}
/**
* Migrate all Collections.
*
* @return void
*/
protected function migrateCollections(): void
{
foreach ($this->collections as $collection) {
$id = $collection['$id'];
Console::log("- {$id}");
$this->createNewMetaData($id);
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
switch ($id) {
case 'attributes':
case 'indexes':
try {
/**
* Create 'databaseInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'databaseId');
} catch (\Throwable $th) {
Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'databaseInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'databaseInternalId');
} catch (\Throwable $th) {
Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'collectionInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'collectionInternalId');
} catch (\Throwable $th) {
Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_collection' index
*/
@$this->projectDB->deleteIndex($id, '_key_collection');
$this->createIndexFromCollection($this->projectDB, $id, '_key_db_collection');
} catch (\Throwable $th) {
Console::warning("'_key_collection' from {$id}: {$th->getMessage()}");
}
break;
case 'projects':
try {
/**
* Create 'teamInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId');
} catch (\Throwable $th) {
Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
}
break;
case 'platforms':
case 'domains':
try {
/**
* Create 'projectInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId');
} catch (\Throwable $th) {
Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_project' index
*/
@$this->projectDB->deleteIndex($id, '_key_project');
$this->createIndexFromCollection($this->projectDB, $id, '_key_project');
} catch (\Throwable $th) {
Console::warning("'_key_project' from {$id}: {$th->getMessage()}");
}
break;
case 'keys':
try {
/**
* Create 'projectInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId');
} catch (\Throwable $th) {
Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'expire' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'expire');
} catch (\Throwable $th) {
Console::warning("'expire' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_project' index
*/
@$this->projectDB->deleteIndex($id, '_key_project');
$this->createIndexFromCollection($this->projectDB, $id, '_key_project');
} catch (\Throwable $th) {
Console::warning("'_key_project' from {$id}: {$th->getMessage()}");
}
break;
case 'webhooks':
try {
/**
* Create 'signatureKey' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'signatureKey');
} catch (\Throwable $th) {
Console::warning("'signatureKey' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'projectInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'projectInternalId');
} catch (\Throwable $th) {
Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_project' index
*/
@$this->projectDB->deleteIndex($id, '_key_project');
$this->createIndexFromCollection($this->projectDB, $id, '_key_project');
} catch (\Throwable $th) {
Console::warning("'_key_project' from {$id}: {$th->getMessage()}");
}
break;
case 'users':
try {
/**
* Create 'phone' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'phone');
} catch (\Throwable $th) {
Console::warning("'phone' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'phoneVerification' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'phoneVerification');
} catch (\Throwable $th) {
Console::warning("'phoneVerification' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create '_key_phone' index
*/
$this->createIndexFromCollection($this->projectDB, $id, '_key_phone');
} catch (\Throwable $th) {
Console::warning("'_key_phone' from {$id}: {$th->getMessage()}");
}
break;
case 'tokens':
case 'sessions':
try {
/**
* Create 'userInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId');
} catch (\Throwable $th) {
Console::warning("'userInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_user' index
*/
@$this->projectDB->deleteIndex($id, '_key_user');
$this->createIndexFromCollection($this->projectDB, $id, '_key_user');
} catch (\Throwable $th) {
Console::warning("'_key_user' from {$id}: {$th->getMessage()}");
}
break;
case 'memberships':
try {
/**
* Create 'teamInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'teamInternalId');
} catch (\Throwable $th) {
Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'userInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'userInternalId');
} catch (\Throwable $th) {
Console::warning("'userInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_unique' index
*/
@$this->projectDB->deleteIndex($id, '_key_unique');
$this->createIndexFromCollection($this->projectDB, $id, '_key_unique');
} catch (\Throwable $th) {
Console::warning("'_key_unique' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_team' index
*/
@$this->projectDB->deleteIndex($id, '_key_team');
$this->createIndexFromCollection($this->projectDB, $id, '_key_team');
} catch (\Throwable $th) {
Console::warning("'_key_team' from {$id}: {$th->getMessage()}");
}
try {
/**
* Re-Create '_key_user' index
*/
@$this->projectDB->deleteIndex($id, '_key_user');
$this->createIndexFromCollection($this->projectDB, $id, '_key_user');
} catch (\Throwable $th) {
Console::warning("'_key_user' from {$id}: {$th->getMessage()}");
}
break;
}
usleep(50000);
}
}
/**
* Fix run on each document
*
* @param \Utopia\Database\Document $document
* @return \Utopia\Database\Document
*/
protected function fixDocument(Document $document)
{
switch ($document->getCollection()) {
case 'projects':
/**
* Bump Project version number.
*/
$document->setAttribute('version', '0.15.0');
if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) {
$internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId();
$document->setAttribute('teamInternalId', $internalId);
}
break;
case 'keys':
/**
* Add new 'expire' attribute and default to never (0).
*/
if (is_null($document->getAttribute('expire'))) {
$document->setAttribute('expire', 0);
}
/**
* Add Internal ID 'projectId' for Subqueries.
*/
if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) {
$internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId();
$document->setAttribute('projectInternalId', $internalId);
}
break;
case 'audit':
/**
* Add Database Layer to collection resource.
*/
if (str_starts_with($document->getAttribute('resource'), 'collection/')) {
$document
->setAttribute('resource', "database/default/{$document->getAttribute('resource')}")
->setAttribute('event', "databases.default.{$document->getAttribute('event')}");
}
if (str_starts_with($document->getAttribute('resource'), 'document/')) {
$collectionId = explode('.', $document->getAttribute('event'))[1];
$document
->setAttribute('resource', "database/default/collection/{$collectionId}/{$document->getAttribute('resource')}")
->setAttribute('event', "databases.default.{$document->getAttribute('event')}");
}
break;
case 'stats':
/**
* Add Database Layer to stats metric.
*/
if (str_starts_with($document->getAttribute('metric'), 'database.')) {
$metric = ltrim($document->getAttribute('metric'), 'database.');
$document->setAttribute('metric', "databases.default.{$metric}");
}
break;
case 'webhooks':
/**
* Add new 'signatureKey' attribute and generate a random value.
*/
if (empty($document->getAttribute('signatureKey'))) {
$document->setAttribute('signatureKey', \bin2hex(\random_bytes(64)));
}
/**
* Add Internal ID 'projectId' for Subqueries.
*/
if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) {
$internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId();
$document->setAttribute('projectInternalId', $internalId);
}
break;
case 'domains':
/**
* Add Internal ID 'projectId' for Subqueries.
*/
if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) {
$internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId();
$document->setAttribute('projectInternalId', $internalId);
}
break;
case 'tokens':
case 'sessions':
/**
* Add Internal ID 'userId' for Subqueries.
*/
if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) {
$internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId();
$document->setAttribute('userInternalId', $internalId);
}
break;
case 'memberships':
/**
* Add Internal ID 'userId' for Subqueries.
*/
if (!empty($document->getAttribute('userId')) && is_null($document->getAttribute('userInternalId'))) {
$internalId = $this->projectDB->getDocument('users', $document->getAttribute('userId'))->getInternalId();
$document->setAttribute('userInternalId', $internalId);
}
/**
* Add Internal ID 'teamId' for Subqueries.
*/
if (!empty($document->getAttribute('teamId')) && is_null($document->getAttribute('teamInternalId'))) {
$internalId = $this->projectDB->getDocument('teams', $document->getAttribute('teamId'))->getInternalId();
$document->setAttribute('teamInternalId', $internalId);
}
break;
case 'attributes':
case 'indexes':
/**
* Add Internal ID 'collectionId' for Subqueries.
*/
if (!empty($document->getAttribute('collectionId')) && is_null($document->getAttribute('collectionInternalId'))) {
$internalId = $this->projectDB->getDocument('database_1', $document->getAttribute('collectionId'))->getInternalId();
$document->setAttribute('collectionInternalId', $internalId);
}
/**
* Add Internal ID 'databaseInternalId' for Subqueries.
*/
if (is_null($document->getAttribute('databaseInternalId'))) {
$document->setAttribute('databaseInternalId', '1');
}
/**
* Add Internal ID 'databaseInternalId' for Subqueries.
*/
if (is_null($document->getAttribute('databaseId'))) {
$document->setAttribute('databaseId', 'default');
}
try {
/**
* Re-create Collection Document
*/
$this->projectDB->deleteDocument($document->getCollection(), $document->getId());
$this->projectDB->createDocument($document->getCollection(), $document->setAttribute('$id', "1_{$document->getInternalId()}_{$document->getAttribute('key')}"));
} catch (\Throwable $th) {
Console::warning("Create Collection Document - {$th->getMessage()}");
}
$document = null;
break;
case 'platforms':
/**
* Migrate dateCreated to $createdAt.
*/
if (empty($document->getCreatedAt())) {
$document->setAttribute('$createdAt', $document->getAttribute('dateCreated'));
}
/**
* Migrate dateUpdated to $updatedAt.
*/
if (empty($document->getUpdatedAt())) {
$document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated'));
}
/**
* Add Internal ID 'projectId' for Subqueries.
*/
if (!empty($document->getAttribute('projectId')) && is_null($document->getAttribute('projectInternalId'))) {
$internalId = $this->projectDB->getDocument('projects', $document->getAttribute('projectId'))->getInternalId();
$document->setAttribute('projectInternalId', $internalId);
}
break;
case 'buckets':
/**
* Migrate dateCreated to $createdAt.
*/
if (empty($document->getCreatedAt())) {
$document->setAttribute('$createdAt', $document->getAttribute('dateCreated'));
}
/**
* Migrate dateUpdated to $updatedAt.
*/
if (empty($document->getUpdatedAt())) {
$document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated'));
}
/**
* Migrate all Storage Buckets to use Internal ID.
*/
$internalId = $this->projectDB->getDocument('buckets', $document->getId())->getInternalId();
$this->createNewMetaData("bucket_{$internalId}");
/**
* Migrate all Storage Bucket Files.
*/
$this->migrateBucketFiles($document);
break;
case 'users':
/**
* Set 'phoneVerification' to false if not set.
*/
if (is_null($document->getAttribute('phoneVerification'))) {
$document->setAttribute('phoneVerification', false);
}
break;
case 'functions':
/**
* Migrate dateCreated to $createdAt.
*/
if (empty($document->getCreatedAt())) {
$document->setAttribute('$createdAt', $document->getAttribute('dateCreated'));
}
/**
* Migrate dateUpdated to $updatedAt.
*/
if (empty($document->getUpdatedAt())) {
$document->setAttribute('$updatedAt', $document->getAttribute('dateUpdated'));
}
break;
case 'deployments':
case 'executions':
case 'teams':
/**
* Migrate dateCreated to $createdAt.
*/
if (empty($document->getCreatedAt())) {
$document->setAttribute('$createdAt', $document->getAttribute('dateCreated'));
}
break;
}
return $document;
}
/**
* Creates new metadata that was introduced for a collection and enforces the Internal ID.
*
* @param string $id
* @return void
*/
protected function createNewMetaData(string $id, string $to = null): void
{
$to ??= $id;
/**
* Skip files collection.
*/
if (in_array($id, ['files', 'databases'])) {
return;
}
try {
/**
* Replace project UID with Internal ID.
*/
$this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}` RENAME TO `_{$this->project->getInternalId()}_{$to}`")->execute();
} catch (\Throwable $th) {
Console::warning("Migrating {$id} Collection: {$th->getMessage()}");
}
try {
/**
* Replace project UID with Internal ID on permissions table.
*/
$this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getId()}_{$id}_perms` RENAME TO `_{$this->project->getInternalId()}_{$to}_perms`")->execute();
} catch (\Throwable $th) {
Console::warning("Migrating {$id} Collection: {$th->getMessage()}");
}
try {
/**
* Add _createdAt attribute.
*/
$this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_createdAt` int unsigned DEFAULT NULL")->execute();
} catch (\Throwable $th) {
Console::warning("Migrating {$id} Collection: {$th->getMessage()}");
}
try {
/**
* Add _updatedAt attribute.
*/
$this->pdo->prepare("ALTER TABLE `_{$this->project->getInternalId()}_{$to}` ADD COLUMN IF NOT EXISTS `_updatedAt` int unsigned DEFAULT NULL")->execute();
} catch (\Throwable $th) {
Console::warning("Migrating {$id} Collection: {$th->getMessage()}");
}
try {
/**
* Create index for _createdAt.
*/
$this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_created_at` ON `_{$this->project->getInternalId()}_{$to}` (`_createdAt`)")->execute();
} catch (\Throwable $th) {
Console::warning("Migrating {$id} Collection: {$th->getMessage()}");
}
try {
/**
* Create index for _updatedAt.
*/
$this->pdo->prepare("CREATE INDEX IF NOT EXISTS `_updated_at` ON `_{$this->project->getInternalId()}_{$to}` (`_updatedAt`)")->execute();
} catch (\Throwable $th) {
Console::warning("Migrating {$id} Collection: {$th->getMessage()}");
}
}
}

1
tests/unit/Migration/MigrationTest.php
View file

@ -4,7 +4,6 @@ namespace Appwrite\Tests;
use Appwrite\Migration\Migration;
use PHPUnit\Framework\TestCase;
use ReflectionClass;
use ReflectionMethod;
use Utopia\Database\Document;

11
tests/unit/Migration/MigrationV13Test.php
View file

@ -38,15 +38,4 @@ class MigrationV13Test extends MigrationTest
$this->assertEquals($document->getAttribute('events'), ['users.*.create']);
}
// This fails due to event validator update
// public function testEventsConversion()
// {
// $migration = new V13();
// $events = $migration->migrateEvents($migration->events);
// foreach ($events as $event) {
// $this->assertTrue((new Event())->isValid($event), $event);
// }
// $this->assertCount(44, $events);
// }
}

172
tests/unit/Migration/MigrationV14Test.php Normal file
View file

@ -0,0 +1,172 @@
<?php
namespace Appwrite\Tests;
use ReflectionClass;
use Appwrite\Migration\Version\V14;
use Utopia\Database\Document;
class MigrationV14Test extends MigrationTest
{
public function setUp(): void
{
$this->migration = new V14();
$reflector = new ReflectionClass('Appwrite\Migration\Version\V14');
$this->method = $reflector->getMethod('fixDocument');
$this->method->setAccessible(true);
}
public function testMigrateProjects()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'projects',
'version' => '0.14.0'
]));
$this->assertEquals($document->getAttribute('version'), '0.15.0');
$this->assertEquals($document->getAttribute('version'), '0.15.0');
}
public function testMigrateKeys()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'keys'
]));
$this->assertArrayHasKey('expire', $document->getArrayCopy());
$this->assertEquals($document->getAttribute('expire'), 0);
}
public function testMigrateWebhooks()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'webhooks'
]));
$this->assertArrayHasKey('signatureKey', $document->getArrayCopy());
$this->assertEquals(strlen($document->getAttribute('signatureKey')), 128);
}
public function testMigrateUsers()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'users',
'phoneVerification' => null
]));
$this->assertArrayHasKey('phoneVerification', $document->getArrayCopy());
$this->assertFalse($document->getAttribute('phoneVerification'));
}
public function testMigratePlatforms()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'platforms',
'$createdAt' => null,
'$updatedAt' => null,
'dateCreated' => 123456789,
'dateUpdated' => 987654321
]));
$this->assertEquals($document->getCreatedAt(), 123456789);
$this->assertEquals($document->getUpdatedAt(), 987654321);
}
public function testMigrateFunctions()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'functions',
'$createdAt' => null,
'$updatedAt' => null,
'dateCreated' => 123456789,
'dateUpdated' => 987654321
]));
$this->assertEquals($document->getCreatedAt(), 123456789);
$this->assertEquals($document->getUpdatedAt(), 987654321);
}
public function testMigrateDeployments()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'deployments',
'$createdAt' => null,
'dateCreated' => 123456789,
]));
$this->assertEquals($document->getCreatedAt(), 123456789);
}
public function testMigrateExecutions()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'executions',
'$createdAt' => null,
'dateCreated' => 123456789,
]));
$this->assertEquals($document->getCreatedAt(), 123456789);
}
public function testMigrateTeams()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'teams',
'$createdAt' => null,
'dateCreated' => 123456789,
]));
$this->assertEquals($document->getCreatedAt(), 123456789);
}
public function testMigrateAudits()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'audit',
'resource' => 'collection/movies',
'event' => 'collections.movies.create'
]));
$this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies');
$this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.create');
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'audit',
'resource' => 'document/avatar',
'event' => 'collections.movies.documents.avatar.create'
]));
$this->assertEquals($document->getAttribute('resource'), 'database/default/collection/movies/document/avatar');
$this->assertEquals($document->getAttribute('event'), 'databases.default.collections.movies.documents.avatar.create');
}
public function testMigrateStats()
{
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'stats',
'metric' => 'database.collections.62b2039844d4277495d0.documents.create'
]));
$this->assertEquals($document->getAttribute('metric'), 'databases.default.collections.62b2039844d4277495d0.documents.create');
$document = $this->fixDocument(new Document([
'$id' => 'appwrite',
'$collection' => 'stats',
'metric' => 'users.create'
]));
$this->assertEquals($document->getAttribute('metric'), 'users.create');
}
}