diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index ca56e080a3..fbe0a30359 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -111,7 +111,7 @@ function createAttribute(string $collectionId, Document $attribute, Response $re } $dbForProject->deleteCachedDocument('collections', $collectionId); - $dbForProject->deleteCachedCollection('collection_' . $collectionId); + $dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId()); // Pass clone of $attribute object to workers // so we can later modify Document to fit response model @@ -166,7 +166,7 @@ App::post('/v1/database/collections') $collectionId = $collectionId == 'unique()' ? $dbForProject->getId() : $collectionId; try { - $collection = $dbForProject->createDocument('collections', new Document([ + $dbForProject->createDocument('collections', new Document([ '$id' => $collectionId, '$read' => $read ?? [], // Collection permissions for collection documents (based on permission model) '$write' => $write ?? [], // Collection permissions for collection documents (based on permission model) @@ -177,8 +177,9 @@ App::post('/v1/database/collections') 'name' => $name, 'search' => implode(' ', [$collectionId, $name]), ])); + $collection = $dbForProject->getDocument('collections', $collectionId); - $dbForProject->createCollection('collection_' . $collectionId); + $dbForProject->createCollection('collection_' . $collection->getInternalId()); } catch (DuplicateException $th) { throw new Exception('Collection already exists', 409, Exception::COLLECTION_ALREADY_EXISTS); } catch (LimitException $th) { @@ -402,7 +403,8 @@ App::get('/v1/database/:collectionId/usage') /** @var Utopia\Database\Database $dbForProject */ /** @var Utopia\Registry\Registry $register */ - $collection = $dbForProject->getCollection('collection_' . $collectionId); + $collectionDocument = $dbForProject->getDocument('collections', $collectionId); + $collection = $dbForProject->getCollection('collection_' . $collectionDocument->getInternalId()); if ($collection->isEmpty()) { throw new Exception('Collection not found', 404, Exception::COLLECTION_NOT_FOUND); @@ -513,7 +515,8 @@ App::get('/v1/database/collections/:collectionId/logs') /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ - $collection = $dbForProject->getCollection('collection_' . $collectionId); + $collectionDocument = $dbForProject->getDocument('collections', $collectionId); + $collection = $dbForProject->getCollection('collection_' . $collectionDocument->getInternalId()); if ($collection->isEmpty()) { throw new Exception('Collection not found', 404, Exception::COLLECTION_NOT_FOUND); @@ -676,7 +679,7 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Failed to remove collection from DB', 500, Exception::GENERAL_SERVER_ERROR); } - $dbForProject->deleteCachedCollection('collection_' . $collectionId); + $dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId()); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) @@ -1259,7 +1262,7 @@ App::delete('/v1/database/collections/:collectionId/attributes/:key') } $dbForProject->deleteCachedDocument('collections', $collectionId); - $dbForProject->deleteCachedCollection('collection_' . $collectionId); + $dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId()); $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) @@ -1642,9 +1645,9 @@ App::post('/v1/database/collections/:collectionId/documents') try { if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ - $document = Authorization::skip(fn() => $dbForProject->createDocument('collection_' . $collectionId, new Document($data))); + $document = Authorization::skip(fn() => $dbForProject->createDocument('collection_' . $collection->getInternalId(), new Document($data))); } else { - $document = $dbForProject->createDocument('collection_' . $collectionId, new Document($data)); + $document = $dbForProject->createDocument('collection_' . $collection->getInternalId(), new Document($data)); } $document->setAttribute('$collection', $collectionId); } @@ -1742,8 +1745,8 @@ App::get('/v1/database/collections/:collectionId/documents') $cursorDocument = null; if (!empty($cursor)) { $cursorDocument = $collection->getAttribute('permission') === 'collection' - ? Authorization::skip(fn () => $dbForProject->getDocument('collection_' . $collectionId, $cursor)) - : $dbForProject->getDocument('collection_' . $collectionId, $cursor); + ? Authorization::skip(fn () => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $cursor)) + : $dbForProject->getDocument('collection_' . $collection->getInternalId(), $cursor); if ($cursorDocument->isEmpty()) { throw new Exception("Document '{$cursor}' for the 'cursor' value not found.", 400, Exception::GENERAL_CURSOR_NOT_FOUND); @@ -1752,11 +1755,11 @@ App::get('/v1/database/collections/:collectionId/documents') if ($collection->getAttribute('permission') === 'collection') { /** @var Document[] $documents */ - $documents = Authorization::skip(fn() => $dbForProject->find('collection_' . $collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection)); - $sum = Authorization::skip(fn() => $dbForProject->count('collection_' . $collectionId, $queries, APP_LIMIT_COUNT)); + $documents = Authorization::skip(fn() => $dbForProject->find('collection_' . $collection->getInternalId(), $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection)); + $sum = Authorization::skip(fn() => $dbForProject->count('collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT)); } else { - $documents = $dbForProject->find('collection_' . $collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection); - $sum = $dbForProject->count('collection_' . $collectionId, $queries, APP_LIMIT_COUNT); + $documents = $dbForProject->find('collection_' . $collection->getInternalId(), $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection); + $sum = $dbForProject->count('collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); } /** @@ -1818,9 +1821,9 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ - $document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collectionId, $documentId)); + $document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId)); } else { - $document = $dbForProject->getDocument('collection_' . $collectionId, $documentId); + $document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId); } /** @@ -1872,7 +1875,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId/logs') throw new Exception('Collection not found', 404, Exception::COLLECTION_NOT_FOUND); } - $document = $dbForProject->getDocument('collection_' . $collectionId, $documentId); + $document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId); if ($document->isEmpty()) { throw new Exception('No document found', 404, Exception::DOCUMENT_NOT_FOUND); @@ -1981,9 +1984,9 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); } - $document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collectionId, $documentId)); + $document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId)); } else { - $document = $dbForProject->getDocument('collection_' . $collectionId, $documentId); + $document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId); } @@ -2031,9 +2034,9 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') try { if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ - $document = Authorization::skip(fn() => $dbForProject->updateDocument('collection_' . $collection->getId(), $document->getId(), new Document($data))); + $document = Authorization::skip(fn() => $dbForProject->updateDocument('collection_' . $collection->getInternalId(), $document->getId(), new Document($data))); } else { - $document = $dbForProject->updateDocument('collection_' . $collection->getId(), $document->getId(), new Document($data)); + $document = $dbForProject->updateDocument('collection_' . $collection->getInternalId(), $document->getId(), new Document($data)); } /** * Reset $collection attribute to remove prefix. @@ -2116,9 +2119,9 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ - $document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collectionId, $documentId)); + $document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId)); } else { - $document = $dbForProject->getDocument('collection_' . $collectionId, $documentId); + $document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId); } if ($document->isEmpty()) { @@ -2126,12 +2129,12 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') } if ($collection->getAttribute('permission') === 'collection') { - Authorization::skip(fn() => $dbForProject->deleteDocument('collection_' . $collectionId, $documentId)); + Authorization::skip(fn() => $dbForProject->deleteDocument('collection_' . $collection->getInternalId(), $documentId)); } else { - $dbForProject->deleteDocument('collection_' . $collectionId, $documentId); + $dbForProject->deleteDocument('collection_' . $collection->getInternalId(), $documentId); } - $dbForProject->deleteCachedDocument('collection_' . $collectionId, $documentId); + $dbForProject->deleteCachedDocument('collection_' . $collection->getInternalId(), $documentId); /** * Reset $collection attribute to remove prefix. diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index d709c29c05..ce3246f4fa 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -101,10 +101,10 @@ App::post('/v1/projects') 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), ])); + /** @var array $collections */ + $collections = Config::getParam('collections', []); - $collections = Config::getParam('collections', []); /** @var array $collections */ - - $dbForProject->setNamespace('_project_' . $project->getId()); + $dbForProject->setNamespace("_{$project->getId()}"); $dbForProject->create('appwrite'); $audit = new Audit($dbForProject); @@ -269,7 +269,7 @@ App::get('/v1/projects/:projectId/usage') ], ]; - $dbForProject->setNamespace('_project_' . $projectId); + $dbForProject->setNamespace("_{$projectId}"); $metrics = [ 'requests', diff --git a/app/init.php b/app/init.php index cefbe25738..75153ce554 100644 --- a/app/init.php +++ b/app/init.php @@ -780,7 +780,7 @@ App::setResource('dbForProject', function($db, $cache, $project) { $database = new Database(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_project_'.$project->getId()); + $database->setNamespace("_{$project->getId()}"); return $database; }, ['db', 'cache', 'project']); @@ -790,7 +790,7 @@ App::setResource('dbForConsole', function($db, $cache) { $database = new Database(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_project_console'); + $database->setNamespace('_console'); return $database; }, ['db', 'cache']); diff --git a/app/realtime.php b/app/realtime.php index b2ddf67634..d72af7f606 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -137,7 +137,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume */ go(function () use ($register, $containerId, &$statsDocument, $logError) { try { - [$database, $returnDatabase] = getDatabase($register, '_project_console'); + [$database, $returnDatabase] = getDatabase($register, '_console'); $document = new Document([ '$id' => $database->getId(), '$collection' => 'realtime', @@ -190,7 +190,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } try { - [$database, $returnDatabase] = getDatabase($register, '_project_console'); + [$database, $returnDatabase] = getDatabase($register, '_console'); $statsDocument ->setAttribute('timestamp', time()) @@ -217,7 +217,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, */ if ($realtime->hasSubscriber('console', 'role:member', 'project')) { - [$database, $returnDatabase] = getDatabase($register, '_project_console'); + [$database, $returnDatabase] = getDatabase($register, '_console'); $payload = []; @@ -321,7 +321,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, return; } - [$database, $returnDatabase] = getDatabase($register, '_project_' . $projectId); + [$database, $returnDatabase] = getDatabase($register, "_{$projectId}"); $user = $database->getDocument('users', $userId); @@ -397,7 +397,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $cache = new Cache(new RedisCache($redis)); $database = new Database(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_project_' . $project->getId()); + $database->setNamespace("_{$project->getId()}"); /* * Project Check @@ -504,7 +504,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $cache = new Cache(new RedisCache($redis)); $database = new Database(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_project_' . $realtime->connections[$connection]['projectId']); + $database->setNamespace("_{$realtime->connections[$connection]['projectId']}"); /* * Abuse Check diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 416f936d4c..ff0705eb32 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -46,7 +46,13 @@ $cli $offset = 0; $projects = [$console]; $count = 0; - $totalProjects = $consoleDB->count('projects') + 1; + + try { + $totalProjects = $consoleDB->count('projects') + 1; + } catch (\Throwable $th) { + $consoleDB->setNamespace('_console'); + $totalProjects = $consoleDB->count('projects') + 1; + } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; $migration = new $class(); diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 0ee28f5cb6..87a40bc5a8 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -259,7 +259,7 @@ $cli $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $dbForConsole->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $dbForConsole->setNamespace('_project_console'); + $dbForConsole->setNamespace('_console'); $latestTime = []; @@ -325,7 +325,7 @@ $cli $projectId = $point['projectId']; if (!empty($projectId) && $projectId !== 'console') { - $dbForProject->setNamespace('_project_' . $projectId); + $dbForProject->setNamespace('_' . $projectId); $metricUpdated = $metric; if (!empty($groupBy)) { @@ -415,7 +415,7 @@ $cli $projectId = $project->getId(); // Get total storage - $dbForProject->setNamespace('_project_' . $projectId); + $dbForProject->setNamespace('_' . $projectId); $storageTotal = $dbForProject->sum('tags', 'size'); $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes @@ -489,7 +489,7 @@ $cli foreach ($collections as $collection => $options) { try { - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $count = $dbForProject->count($collection); $metricPrefix = $options['metricPrefix'] ?? ''; $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; @@ -545,7 +545,7 @@ $cli $subCollectionTotals = []; //total project level sum of sub collections do { // Loop over all the parent collection document for each sub collection - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $parents = $dbForProject->find($collection, [], 100, cursor: $latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections if (empty($parents)) { @@ -556,12 +556,12 @@ $cli foreach ($parents as $parent) { foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getId()); $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes @@ -610,12 +610,12 @@ $cli continue; } - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getId(), $sum['field']); $subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.total"; $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes @@ -660,7 +660,7 @@ $cli * Inserting project level counts for sub collections like database.documents.count */ foreach ($subCollectionCounts as $subCollection => $count) { - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; @@ -709,7 +709,7 @@ $cli * Inserting project level sums for sub collections like storage.total */ foreach ($subCollectionTotals as $subCollection => $count) { - $dbForProject->setNamespace("_project_{$projectId}"); + $dbForProject->setNamespace("_{$projectId}"); $metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total"; @@ -760,4 +760,4 @@ $cli Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); + }); \ No newline at end of file diff --git a/app/workers/database.php b/app/workers/database.php index 94318e8cec..d95ce122d0 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -86,7 +86,7 @@ class DatabaseV1 extends Worker $project = $dbForConsole->getDocument('projects', $projectId); try { - if(!$dbForProject->createAttribute('collection_' . $collectionId, $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) { + if(!$dbForProject->createAttribute('collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) { throw new Exception('Failed to create Attribute'); } $dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available')); @@ -135,7 +135,7 @@ class DatabaseV1 extends Worker // - failed: attribute was never created // - stuck: attribute was available but cannot be removed try { - if($status !== 'failed' && !$dbForProject->deleteAttribute('collection_' . $collectionId, $key)) { + if($status !== 'failed' && !$dbForProject->deleteAttribute('collection_' . $collection->getInternalId(), $key)) { throw new Exception('Failed to delete Attribute'); } $dbForProject->deleteDocument('attributes', $attribute->getId()); @@ -210,7 +210,7 @@ class DatabaseV1 extends Worker } $dbForProject->deleteCachedDocument('collections', $collectionId); - $dbForProject->deleteCachedCollection('collection_' . $collectionId); + $dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId()); } /** @@ -233,7 +233,7 @@ class DatabaseV1 extends Worker $project = $dbForConsole->getDocument('projects', $projectId); try { - if(!$dbForProject->createIndex('collection_' . $collectionId, $key, $type, $attributes, $lengths, $orders)) { + if(!$dbForProject->createIndex('collection_' . $collection->getInternalId(), $key, $type, $attributes, $lengths, $orders)) { throw new Exception('Failed to create Index'); } $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available')); @@ -276,7 +276,7 @@ class DatabaseV1 extends Worker $project = $dbForConsole->getDocument('projects', $projectId); try { - if($status !== 'failed' && !$dbForProject->deleteIndex('collection_' . $collectionId, $key)) { + if($status !== 'failed' && !$dbForProject->deleteIndex('collection_' . $collection->getInternalId(), $key)) { throw new Exception('Failed to delete index'); } $dbForProject->deleteDocument('indexes', $index->getId()); diff --git a/composer.json b/composer.json index db6c354b62..5bf4f01bf9 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.12.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.14.*", + "utopia-php/database": "0.15.*", "utopia-php/locale": "0.4.*", "utopia-php/orchestration": "0.4.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index e46501080f..1151a1a301 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "71d2e3bdd2ee9ed2bd4c8ae2e62ea1a2", + "content-hash": "318c53aaac3cdfbdb728acc5e59b8057", "packages": [ { "name": "adhocore/jwt", @@ -2141,16 +2141,16 @@ }, { "name": "utopia-php/database", - "version": "0.14.1", + "version": "0.15.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "ecc143f2cfe16b23675407035c6b5375ba263285" + "reference": "eb4f61ec40d697acdfd574638ecd075e4f44b864" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/ecc143f2cfe16b23675407035c6b5375ba263285", - "reference": "ecc143f2cfe16b23675407035c6b5375ba263285", + "url": "https://api.github.com/repos/utopia-php/database/zipball/eb4f61ec40d697acdfd574638ecd075e4f44b864", + "reference": "eb4f61ec40d697acdfd574638ecd075e4f44b864", "shasum": "" }, "require": { @@ -2198,9 +2198,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.14.1" + "source": "https://github.com/utopia-php/database/tree/0.15.1" }, - "time": "2022-01-25T13:01:20+00:00" + "time": "2022-02-22T09:33:37+00:00" }, { "name": "utopia-php/domains", @@ -3087,7 +3087,7 @@ "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator", - "reference": "86a8e495b14c6f112de71fa6c500d36dcded8e95" + "reference": "4a43aa70c2f0b243edfc689f618a85f7d0817287" }, "require": { "ext-curl": "*", @@ -3122,7 +3122,7 @@ } ], "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", - "time": "2022-02-22T06:42:40+00:00" + "time": "2022-02-22T10:51:55+00:00" }, { "name": "composer/pcre", @@ -3697,6 +3697,9 @@ "require": { "php": "^7.1 || ^8.0" }, + "replace": { + "myclabs/deep-copy": "self.version" + }, "require-dev": { "doctrine/collections": "^1.0", "doctrine/common": "^2.6", @@ -6582,5 +6585,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.1.0" } diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 044641335c..2f08e8815f 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -38,6 +38,26 @@ abstract class Migration '0.13.0' => 'V12', ]; + /** + * @var array + */ + protected array $collections; + + public function __construct() + { + $this->collections = array_merge([ + '_metadata' => [ + '$id' => '_metadata' + ], + 'audit' => [ + '$id' => 'audit' + ], + 'abuse' => [ + '$id' => 'abuse' + ] + ], Config::getParam('collections', [])); + } + /** * Set project for migration. * @@ -51,7 +71,7 @@ abstract class Migration { $this->project = $project; $this->projectDB = $projectDB; - $this->projectDB->setNamespace('_project_' . $this->project->getId()); + $this->projectDB->setNamespace('_' . $this->project->getId()); $this->consoleDB = $consoleDB; @@ -67,10 +87,7 @@ abstract class Migration { Runtime::enableCoroutine(SWOOLE_HOOK_ALL); - /** @var array $collections */ - $collections = Config::getParam('collections', []); - - foreach ($collections as $collection) { + foreach ($this->collections as $collection) { $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collection['$id']); diff --git a/src/Appwrite/Migration/Version/V12.php b/src/Appwrite/Migration/Version/V12.php index 9b9376abac..73177b61fb 100644 --- a/src/Appwrite/Migration/Version/V12.php +++ b/src/Appwrite/Migration/Version/V12.php @@ -4,17 +4,178 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\Document; class V12 extends Migration { + /** + * @var \PDO $pdo + */ + private $pdo; + public function execute(): void { + global $register; Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + $this->pdo = $register->get('db'); + + Console::info('Migrating Project Schemas'); + $this->migrateProjectSchema($this->project->getId()); + + /** + * Switch to migrated Console Project + */ + if ($this->project->getId() === 'console') { + $this->consoleDB->setNamespace('_console'); + $this->projectDB->setNamespace('_console'); + } + + Console::info('Migrating Permissions'); + $this->fixPermissions(); + Console::info('Migrating Collections'); + $this->fixCollections(); + Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); } + /** + * Migrate Project Tables. + * + * @param string $projectId + * @return void + * @throws \Exception + * @throws \PDOException + */ + private function migrateProjectSchema(string $projectId): void + { + /** + * Remove empty generated Console Project. + */ + if ($this->consoleDB->getNamespace() === '_project_console' && $projectId === 'console') { + $all = []; + foreach ($this->collections as $collection) { + $all[] = "_{$projectId}_{$collection['$id']}"; + $all[] = "_{$projectId}_{$collection['$id']}_perms"; + } + $this->pdo->prepare('DROP TABLE IF EXISTS ' . implode(', ', $all) . ';')->execute(); + } elseif ($this->projectDB->getNamespace() === '_console') { + return; + } + + /** + * Rename Database Tables. + */ + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + $this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_{$id} RENAME TO _{$projectId}_{$id}")->execute(); + $this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_{$id}_perms ( + `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `_type` VARCHAR(12) NOT NULL, + `_permission` VARCHAR(255) NOT NULL, + `_document` VARCHAR(255) NOT NULL, + PRIMARY KEY (`_id`), + UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`), + INDEX `_index2` (`_permission`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute(); + } + } + + /** + * Migrate all Collection Structure. + * + * @return void + */ + protected function fixCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + Console::log("- {$id}"); + switch ($id) { + case 'sessions': + try { + $this->projectDB->renameAttribute($id, 'providerToken', 'providerAccessToken'); + } catch (\Throwable $th) { + Console::warning("'providerAccessToken' from {$id}: {$th->getMessage()}"); + } + try { + $this->projectDB->createAttribute(collection: $id, id: 'providerRefreshToken', type: Database::VAR_STRING, size: 16384, signed: true, required: true, filters: ['encrypt']); + } catch (\Throwable $th) { + Console::warning("'providerRefreshToken' from {$id}: {$th->getMessage()}"); + } + try { + $this->projectDB->createAttribute(collection: $id, id: 'providerAccessTokenExpiry', type: Database::VAR_INTEGER, size: 0, required: true); + } catch (\Throwable $th) { + Console::warning("'providerAccessTokenExpiry' from {$id}: {$th->getMessage()}"); + } + break; + } + usleep(100000); + } + } + + /** + * Migrate all Permission to new System with dedicated Table. + * @return void + * @throws \Exception + */ + protected function fixPermissions() + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + Console::log("- {$collection['$id']}"); + $nextDocument = null; + + do { + $documents = $this->projectDB->find($id, limit: $this->limit, cursor: $nextDocument); + $count = count($documents); + + \Co\run(function (array $documents) { + foreach ($documents as $document) { + go(function (Document $document) { + $sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}` WHERE _uid = {$this->pdo->quote($document->getid())}"; + $stmt = $this->pdo->prepare($sql); + $stmt->execute(); + + $permissions = $stmt->fetch(); + + $read = json_decode($permissions['_read'] ?? null) ?? []; + $write = json_decode($permissions['_write'] ?? null) ?? []; + + $permissions = []; + foreach ($read as $permission) { + $permissions[] = "('read', '{$permission}', '{$document->getId()}')"; + } + + foreach ($write as $permission) { + $permissions[] = "('write', '{$permission}', '{$document->getId()}')"; + } + + if (!empty($permissions)) { + $queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions); + $stmtPermissions = $this->pdo->prepare($queryPermissions); + $stmtPermissions->execute(); + } + }, $document); + } + }, $documents); + + if ($count !== $this->limit) { + $nextDocument = null; + } else { + $nextDocument = end($documents); + } + } while (!is_null($nextDocument)); + } + + /** + * Timeout to give MariaDB some room to breath + */ + usleep(100000); + } + protected function fixDocument(Document $document) { switch ($document->getCollection()) { @@ -92,6 +253,15 @@ class V12 extends Migration } break; + + case 'sessions': + $document + ->setAttribute('providerRefreshToken', '') + ->setAttribute('providerAccessTokenExpiry', 0) + ->setAttribute('providerAccessToken', $document->getAttribute('providerToken', '')) + ->removeAttribute('providerToken'); + + break; } return $document; diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index b9ed5af275..d9bad8b90c 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -178,10 +178,10 @@ abstract class Worker if (!$projectId) { throw new \Exception('ProjectID not provided - cannot get database'); } - $namespace = "_project_{$projectId}"; + $namespace = "_{$projectId}"; break; case self::DATABASE_CONSOLE: - $namespace = "_project_console"; + $namespace = "_console"; $sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created break; default: diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 3084ff5e28..cf2887081d 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -149,7 +149,21 @@ trait UsersBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'search' => 'manchester-united.co.uk' + 'search' => 'united.co.uk' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertIsArray($response['body']); + $this->assertIsArray($response['body']['users']); + $this->assertIsInt($response['body']['sum']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertCount(1, $response['body']['users']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'man' ]); $this->assertEquals($response['headers']['status-code'], 200); @@ -165,6 +179,7 @@ trait UsersBase ], $this->getHeaders()), [ 'search' => $data['userId'] ]); + $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']);