From 5678100d7dbd24a3603e9cdb463806ae1a429a70 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 29 Jul 2021 19:33:57 -0400 Subject: [PATCH 1/9] Use main branch of utopia-php/database --- composer.json | 7 ++++++- composer.lock | 22 ++++++++-------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index 8467da8097..d48d2378aa 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,12 @@ "adhocore/jwt": "1.1.2", "slickdeals/statsd": "3.1.0" }, - "repositories": [], + "repositories": [ + { + "type": "git", + "url": "https://github.com/utopia-php/database" + } + ], "require-dev": { "appwrite/sdk-generator": "0.13.0", "swoole/ide-helper": "4.6.7", diff --git a/composer.lock b/composer.lock index 792a6214d2..a5f2a3e062 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "aac413429914bd9299a7ae722f00cef8", + "content-hash": "175f077512c575216c4c88f1d33c6d00", "packages": [ { "name": "adhocore/jwt", @@ -1987,15 +1987,9 @@ "version": "dev-feat-adjusted-query-validator", "source": { "type": "git", - "url": "https://github.com/utopia-php/database.git", + "url": "https://github.com/utopia-php/database", "reference": "cb73391371f70ddb54bc0000064b15c5f173cb7a" }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/cb73391371f70ddb54bc0000064b15c5f173cb7a", - "reference": "cb73391371f70ddb54bc0000064b15c5f173cb7a", - "shasum": "" - }, "require": { "ext-mongodb": "*", "ext-pdo": "*", @@ -2017,7 +2011,11 @@ "Utopia\\Database\\": "src/Database" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Database" + } + }, "license": [ "MIT" ], @@ -2039,10 +2037,6 @@ "upf", "utopia" ], - "support": { - "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-adjusted-query-validator" - }, "time": "2021-08-23T14:18:47+00:00" }, { @@ -6287,5 +6281,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.0.0" + "plugin-api-version": "2.1.0" } From 1c1312baae1fc108945565ccfe54c1d5d7a7cfc8 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 29 Jul 2021 19:36:46 -0400 Subject: [PATCH 2/9] Catch exception for index limit --- app/controllers/api/database.php | 2 +- .../Database/DatabaseCustomServerTest.php | 73 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index a6f4853ee9..cb53dbda9a 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -22,6 +22,7 @@ use Utopia\Database\Validator\Structure; use Utopia\Database\Validator\UID; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Duplicate as DuplicateException; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Structure as StructureException; use Appwrite\Utopia\Response; use Appwrite\Database\Validator\CustomId; @@ -910,7 +911,6 @@ App::post('/v1/database/collections/:collectionId/indexes') $lengths[$key] = ($attributeType === Database::VAR_STRING) ? $attributeSize : null; } - // TODO@kodumbeats should $lengths be a part of the response model? try { $index = $dbForInternal->createDocument('indexes', new Document([ '$id' => $collectionId.'_'.$indexId, diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index cce5080cff..720ea301bc 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -306,4 +306,77 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($response['headers']['status-code'], 404); } + + public function testIndexLimitException() + { + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'testLimitException', + 'read' => ['role:all'], + 'write' => ['role:all'], + ]); + + $this->assertEquals($collection['headers']['status-code'], 201); + $this->assertEquals($collection['body']['name'], 'testLimitException'); + + $collectionId = $collection['body']['$id']; + + // add unique attributes for indexing + for ($i=0; $i < 64; $i++) { + // $this->assertEquals(true, static::getDatabase()->createAttribute('indexLimit', "test{$i}", Database::VAR_STRING, 16, true)); + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'id' => "attribute{$i}", + 'type' => 'string', + 'size' => 64, + 'required' => true, + ]); + + $this->assertEquals($attribute['headers']['status-code'], 201); + } + + sleep(2); + + // testing for indexLimit = 64 + // MariaDB, MySQL, and MongoDB create 3 indexes per new collection + // Add up to the limit, then check if the next index throws IndexLimitException + for ($i=0; $i < 61; $i++) { + // $this->assertEquals(true, static::getDatabase()->createIndex('indexLimit', "index{$i}", Database::INDEX_KEY, ["test{$i}"], [16])); + $index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'id' => "key_attribute{$i}", + 'type' => 'fulltext', + 'attributes' => ["attribute{$i}"], + ]); + } + + sleep(2); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'id' => 'titleIndex', + 'type' => 'fulltext', + 'attributes' => ['attribute62'], + ]); + + $this->assertEquals(400, $tooMany['headers']['status-code']); + } } \ No newline at end of file From 34b6894be2e3ead2701172ec325fb9a4e97b25e7 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 30 Jul 2021 14:40:37 -0400 Subject: [PATCH 3/9] Add more tests for index limits --- .../Database/DatabaseCustomServerTest.php | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 720ea301bc..1fb071bff3 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -341,7 +341,24 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($attribute['headers']['status-code'], 201); } - sleep(2); + sleep(5); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($collection['headers']['status-code'], 200); + $this->assertEquals($collection['body']['name'], 'testLimitException'); + $this->assertIsArray($collection['body']['attributes']); + $this->assertIsArray($collection['body']['indexes']); + $this->assertIsArray($collection['body']['attributesInQueue']); + $this->assertIsArray($collection['body']['indexesInQueue']); + $this->assertCount(64, $collection['body']['attributes']); + $this->assertCount(0, $collection['body']['indexes']); + $this->assertCount(0, $collection['body']['attributesInQueue']); + $this->assertCount(0, $collection['body']['indexesInQueue']); // testing for indexLimit = 64 // MariaDB, MySQL, and MongoDB create 3 indexes per new collection @@ -354,12 +371,15 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'id' => "key_attribute{$i}", - 'type' => 'fulltext', + 'type' => 'key', 'attributes' => ["attribute{$i}"], ]); + + $this->assertEquals(201, $index['headers']['status-code']); + $this->assertEquals("key_attribute{$i}", $index['body']['$id']); } - sleep(2); + sleep(5); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', @@ -367,13 +387,24 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ])); + $this->assertEquals($collection['headers']['status-code'], 200); + $this->assertEquals($collection['body']['name'], 'testLimitException'); + $this->assertIsArray($collection['body']['attributes']); + $this->assertIsArray($collection['body']['indexes']); + $this->assertIsArray($collection['body']['attributesInQueue']); + $this->assertIsArray($collection['body']['indexesInQueue']); + $this->assertCount(0, $collection['body']['attributesInQueue']); + $this->assertCount(0, $collection['body']['indexesInQueue']); + $this->assertCount(64, $collection['body']['attributes']); + $this->assertCount(61, $collection['body']['indexes']); + $tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => 'titleIndex', - 'type' => 'fulltext', + 'id' => 'tooMany', + 'type' => 'key', 'attributes' => ['attribute62'], ]); From bf4c1c8c9c553ef9e84140c47620a6ea00d6478f Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 30 Jul 2021 15:49:53 -0400 Subject: [PATCH 4/9] Fix tests --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 1fb071bff3..39da0eb96b 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -341,7 +341,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($attribute['headers']['status-code'], 201); } - sleep(5); + sleep(10); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', @@ -379,7 +379,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals("key_attribute{$i}", $index['body']['$id']); } - sleep(5); + sleep(10); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', From c072a07ed50b106dce16bd08795b4535498060a4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 2 Aug 2021 14:20:08 -0400 Subject: [PATCH 5/9] Increase sleep for index tests --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 39da0eb96b..30ad33b4c1 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -379,7 +379,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals("key_attribute{$i}", $index['body']['$id']); } - sleep(10); + sleep(20); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', From a4486fd99ecf7c7565a4e1187f968de106a13e9d Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 11 Aug 2021 13:24:24 -0400 Subject: [PATCH 6/9] Use indexId instead of id --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 30ad33b4c1..6be36742d8 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -370,7 +370,7 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => "key_attribute{$i}", + 'indexId' => "key_attribute{$i}", 'type' => 'key', 'attributes' => ["attribute{$i}"], ]); From e6d4897acce5c8bd2c5eee1e9ec7bf1ef9527fe0 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 11 Aug 2021 13:25:22 -0400 Subject: [PATCH 7/9] Debug index limits tests --- .../Database/DatabaseCustomServerTest.php | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 6be36742d8..a3049fa4bd 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -314,6 +314,7 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ + 'collectionId' => 'testLimitException', 'name' => 'testLimitException', 'read' => ['role:all'], 'write' => ['role:all'], @@ -327,21 +328,23 @@ class DatabaseCustomServerTest extends Scope // add unique attributes for indexing for ($i=0; $i < 64; $i++) { // $this->assertEquals(true, static::getDatabase()->createAttribute('indexLimit', "test{$i}", Database::VAR_STRING, 16, true)); - $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes', array_merge([ + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => "attribute{$i}", - 'type' => 'string', + 'attributeId' => "attribute{$i}", 'size' => 64, 'required' => true, ]); $this->assertEquals($attribute['headers']['status-code'], 201); + + // sleep(4); + // \usleep(250000); } - sleep(10); + sleep(20); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', @@ -349,16 +352,18 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ])); + var_dump($collection['body']); + $this->assertEquals($collection['headers']['status-code'], 200); $this->assertEquals($collection['body']['name'], 'testLimitException'); $this->assertIsArray($collection['body']['attributes']); $this->assertIsArray($collection['body']['indexes']); $this->assertIsArray($collection['body']['attributesInQueue']); $this->assertIsArray($collection['body']['indexesInQueue']); - $this->assertCount(64, $collection['body']['attributes']); - $this->assertCount(0, $collection['body']['indexes']); $this->assertCount(0, $collection['body']['attributesInQueue']); $this->assertCount(0, $collection['body']['indexesInQueue']); + $this->assertCount(64, $collection['body']['attributes']); + $this->assertCount(0, $collection['body']['indexes']); // testing for indexLimit = 64 // MariaDB, MySQL, and MongoDB create 3 indexes per new collection @@ -377,6 +382,9 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(201, $index['headers']['status-code']); $this->assertEquals("key_attribute{$i}", $index['body']['$id']); + + // sleep(4); + // \usleep(250000); } sleep(20); @@ -387,6 +395,8 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ])); + var_dump($collection); + $this->assertEquals($collection['headers']['status-code'], 200); $this->assertEquals($collection['body']['name'], 'testLimitException'); $this->assertIsArray($collection['body']['attributes']); From 2f26f8ae4f1e9d5d48b12d65efab51a8453258c3 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 24 Aug 2021 19:35:32 -0400 Subject: [PATCH 8/9] Count index from internal index table --- app/controllers/api/database.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index cb53dbda9a..dd275c9c2e 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -14,6 +14,7 @@ use Utopia\Validator\JSON; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; +use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Validator\Key; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\QueryValidator; @@ -887,6 +888,16 @@ App::post('/v1/database/collections/:collectionId/indexes') throw new Exception('Collection not found', 404); } + $count = $dbForInternal->count('indexes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], 61); + + $limit = 64 - MariaDB::getNumberOfDefaultIndexes(); + + if ($count >= $limit) { + throw new Exception('Index limit exceeded', 400); + } + // Convert Document[] to array of attribute metadata $oldAttributes = \array_map(function ($a) { return $a->getArrayCopy(); @@ -923,7 +934,7 @@ App::post('/v1/database/collections/:collectionId/indexes') 'orders' => $orders, ])); } catch (DuplicateException $th) { - throw new Exception('Attribute already exists', 409); + throw new Exception('Index already exists', 409); } $dbForInternal->purgeDocument('collections', $collectionId); From 0d2bbb0ae97f52fbd770797aa224e5e7547e659d Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 24 Aug 2021 19:35:43 -0400 Subject: [PATCH 9/9] Fix tests for index limits --- .../Database/DatabaseCustomServerTest.php | 30 +++++-------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index a3049fa4bd..e5a75fa685 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -318,6 +318,7 @@ class DatabaseCustomServerTest extends Scope 'name' => 'testLimitException', 'read' => ['role:all'], 'write' => ['role:all'], + 'permission' => 'document', ]); $this->assertEquals($collection['headers']['status-code'], 201); @@ -339,12 +340,9 @@ class DatabaseCustomServerTest extends Scope ]); $this->assertEquals($attribute['headers']['status-code'], 201); - - // sleep(4); - // \usleep(250000); } - sleep(20); + sleep(5); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', @@ -352,16 +350,10 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ])); - var_dump($collection['body']); - $this->assertEquals($collection['headers']['status-code'], 200); $this->assertEquals($collection['body']['name'], 'testLimitException'); $this->assertIsArray($collection['body']['attributes']); $this->assertIsArray($collection['body']['indexes']); - $this->assertIsArray($collection['body']['attributesInQueue']); - $this->assertIsArray($collection['body']['indexesInQueue']); - $this->assertCount(0, $collection['body']['attributesInQueue']); - $this->assertCount(0, $collection['body']['indexesInQueue']); $this->assertCount(64, $collection['body']['attributes']); $this->assertCount(0, $collection['body']['indexes']); @@ -381,13 +373,10 @@ class DatabaseCustomServerTest extends Scope ]); $this->assertEquals(201, $index['headers']['status-code']); - $this->assertEquals("key_attribute{$i}", $index['body']['$id']); - - // sleep(4); - // \usleep(250000); + $this->assertEquals("key_attribute{$i}", $index['body']['key']); } - sleep(20); + sleep(5); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', @@ -395,16 +384,10 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ])); - var_dump($collection); - $this->assertEquals($collection['headers']['status-code'], 200); $this->assertEquals($collection['body']['name'], 'testLimitException'); $this->assertIsArray($collection['body']['attributes']); $this->assertIsArray($collection['body']['indexes']); - $this->assertIsArray($collection['body']['attributesInQueue']); - $this->assertIsArray($collection['body']['indexesInQueue']); - $this->assertCount(0, $collection['body']['attributesInQueue']); - $this->assertCount(0, $collection['body']['indexesInQueue']); $this->assertCount(64, $collection['body']['attributes']); $this->assertCount(61, $collection['body']['indexes']); @@ -413,11 +396,12 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => 'tooMany', + 'indexId' => 'tooMany', 'type' => 'key', - 'attributes' => ['attribute62'], + 'attributes' => ['attribute61'], ]); $this->assertEquals(400, $tooMany['headers']['status-code']); + $this->assertEquals('Index limit exceeded', $tooMany['body']['message']); } } \ No newline at end of file