diff --git a/.gitignore b/.gitignore index d0b2a74730..224a970df4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ /tests/resources/storage/ /.idea/ .DS_Store -.php_cs.cache \ No newline at end of file +.php_cs.cache +debug/ \ No newline at end of file diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 55487eee52..eb6bd5ed4d 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -235,7 +235,7 @@ App::delete('/v1/database/collections/:collectionId') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_NONE) ->param('collectionId', '', new UID(), 'Collection unique ID.') - ->action(function ($collectionId, $response, $projectDB, $webhooks, $audits) { + ->action(function ($collectionId, $response, $projectDB, $webhooks, $audits, $deletes) { /** @var Appwrite\Utopia\Response $response */ /** @var Appwrite\Database\Database $projectDB */ /** @var Appwrite\Event\Event $webhooks */ @@ -250,7 +250,11 @@ App::delete('/v1/database/collections/:collectionId') if (!$projectDB->deleteDocument($collectionId)) { throw new Exception('Failed to remove collection from DB', 500); } - + + $deletes + ->setParam('document', $collection) + ; + $webhooks ->setParam('payload', $response->output($collection, Response::MODEL_COLLECTION)) ; @@ -262,7 +266,7 @@ App::delete('/v1/database/collections/:collectionId') ; $response->noContent(); - }, ['response', 'projectDB', 'webhooks', 'audits']); + }, ['response', 'projectDB', 'webhooks', 'audits', 'deletes']); App::post('/v1/database/collections/:collectionId/documents') ->desc('Create Document') diff --git a/app/workers/deletes.php b/app/workers/deletes.php index d5f7c9284d..14addd2d18 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -10,31 +10,43 @@ use Appwrite\Database\Database; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Document; +use Appwrite\Database\Validator\Authorization; use Appwrite\Storage\Device\Local; +use Utopia\CLI\Console; use Utopia\Config\Config; class DeletesV1 { public $args = []; + protected $consoleDB = null; + public function setUp(): void { } public function perform() { + $projectId = $this->args['projectId']; $document = $this->args['document']; + $document = new Document($document); - switch ($document->getCollection()) { + switch (strval($document->getCollection())) { case Database::SYSTEM_COLLECTION_PROJECTS: $this->deleteProject($document); break; - case Database::SYSTEM_COLLECTION_USERS: - $this->deleteUser($document); + case Database::SYSTEM_COLLECTION_FUNCTIONS: + $this->deleteFunction($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_USERS: + $this->deleteUser($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_COLLECTIONS: + $this->deleteDocuments($document, $projectId); break; - default: + Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); break; } } @@ -43,50 +55,163 @@ class DeletesV1 { // ... Remove environment for this job } + + protected function deleteDocuments(Document $document, $projectId) + { + $collectionId = $document->getId(); + + // Delete Documents in the deleted collection + $this->deleteByGroup([ + '$collection='.$collectionId + ], $this->getProjectDB($projectId)); + } protected function deleteProject(Document $document) { - global $register; - - $consoleDB = new Database(); - $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); - $consoleDB->setNamespace('app_console'); // Main DB - $consoleDB->setMocks(Config::getParam('collections', [])); - // Delete all DBs - $consoleDB->deleteNamespace($document->getId()); + $this->getConsoleDB()->deleteNamespace($document->getId()); $uploads = new Local(APP_STORAGE_UPLOADS.'/app-'.$document->getId()); $cache = new Local(APP_STORAGE_CACHE.'/app-'.$document->getId()); + // Delete all storage directories $uploads->delete($uploads->getRoot(), true); $cache->delete($cache->getRoot(), true); } - protected function deleteUser(Document $user) + protected function deleteUser(Document $document, $projectId) { - global $projectDB; - - $tokens = $user->getAttribute('tokens', []); + $tokens = $document->getAttribute('tokens', []); foreach ($tokens as $token) { - if (!$projectDB->deleteDocument($token->getId())) { + if (!$this->getProjectDB($projectId)->deleteDocument($token->getId())) { throw new Exception('Failed to remove token from DB', 500); } } - $memberships = $projectDB->getCollection([ - 'limit' => 2000, // TODO add members limit - 'offset' => 0, - 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_MEMBERSHIPS, - 'userId='.$user->getId(), - ], - ]); + // Delete Memberships + $this->deleteByGroup([ + '$collection='.Database::SYSTEM_COLLECTION_MEMBERSHIPS, + 'userId='.$document->getId(), + ], $this->getProjectDB($projectId)); + } - foreach ($memberships as $membership) { - if (!$projectDB->deleteDocument($membership->getId())) { - throw new Exception('Failed to remove team membership from DB', 500); + protected function deleteFunction(Document $document, $projectId) + { + $projectDB = $this->getProjectDB($projectId); + $device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId); + + // Delete Tags + $this->deleteByGroup([ + '$collection='.Database::SYSTEM_COLLECTION_TAGS, + 'functionId='.$document->getId(), + ], $projectDB, function(Document $document) use ($device) { + + if ($device->delete($document->getAttribute('path', ''))) { + Console::success('Delete code tag: '.$document->getAttribute('path', '')); + } + else { + Console::error('Dailed to delete code tag: '.$document->getAttribute('path', '')); + } + }); + + // Delete Executions + $this->deleteByGroup([ + '$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS, + 'functionId='.$document->getId(), + ], $projectDB); + } + + protected function deleteById(Document $document, Database $database, callable $callback = null): bool + { + Authorization::disable(); + + if($database->deleteDocument($document->getId())) { + Console::success('Deleted document "'.$document->getId().'" successfully'); + + if(is_callable($callback)) { + $callback($document); + } + + return true; + } + else { + Console::error('Failed to delete document: '.$document->getId()); + return false; + } + + Authorization::reset(); + } + + protected function deleteByGroup(array $filters, Database $database, callable $callback = null) + { + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while($sum === $limit) { + $chunk++; + + Authorization::disable(); + + $results = $database->getCollection([ + 'limit' => $limit, + 'offset' => 0, + 'orderField' => '$id', + 'orderType' => 'ASC', + 'orderCast' => 'string', + 'filters' => $filters, + ]); + + Authorization::reset(); + + $sum = count($results); + + Console::info('Deleting chunk #'.$chunk.'. Found '.$sum.' documents'); + + foreach ($results as $document) { + $this->deleteById($document, $database, $callback); + $count++; } } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } -} + + /** + * @return Database; + */ + protected function getConsoleDB(): Database + { + global $register; + + if($this->consoleDB === null) { + $this->consoleDB = new Database(); + $this->consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); + $this->consoleDB->setNamespace('app_console'); // Main DB + $this->consoleDB->setMocks(Config::getParam('collections', [])); + } + + return $this->consoleDB; + } + + /** + * @return Database; + */ + protected function getProjectDB($projectId): Database + { + global $register; + + $projectDB = new Database(); + $projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); + $projectDB->setNamespace('app_'.$projectId); // Main DB + $projectDB->setMocks(Config::getParam('collections', [])); + + return $projectDB; + } +} \ No newline at end of file diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index dec0b108ef..2f0bdfc766 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -5,10 +5,119 @@ namespace Tests\E2E\Services\Database; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; +use Tests\E2E\Client; class DatabaseCustomServerTest extends Scope { use DatabaseBase; use ProjectCustom; use SideServer; + + public function testDeleteCollection() + { + /** + * Test for SUCCESS + */ + + // Create collection + $actors = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'Actors', + 'read' => ['*'], + 'write' => ['role:1', 'role:2'], + 'rules' => [ + [ + 'label' => 'First Name', + 'key' => 'firstName', + 'type' => 'text', + 'default' => '', + 'required' => true, + 'array' => false + ], + [ + 'label' => 'Last Name', + 'key' => 'lastName', + 'type' => 'text', + 'default' => '', + 'required' => true, + 'array' => false + ], + ], + ]); + + $this->assertEquals($actors['headers']['status-code'], 201); + $this->assertEquals($actors['body']['$collection'], 0); + $this->assertEquals($actors['body']['name'], 'Actors'); + $this->assertIsArray($actors['body']['$permissions']); + $this->assertIsArray($actors['body']['$permissions']['read']); + $this->assertIsArray($actors['body']['$permissions']['write']); + $this->assertCount(1, $actors['body']['$permissions']['read']); + $this->assertCount(2, $actors['body']['$permissions']['write']); + + // Add Documents to the collection + $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'firstName' => 'Tom', + 'lastName' => 'Holland', + ], + 'read' => ['user:'.$this->getUser()['$id']], + 'write' => ['user:'.$this->getUser()['$id']], + ]); + + $document2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'firstName' => 'Samuel', + 'lastName' => 'Jackson', + ], + 'read' => ['user:'.$this->getUser()['$id']], + 'write' => ['user:'.$this->getUser()['$id']], + ]); + + $this->assertEquals($document1['headers']['status-code'], 201); + $this->assertEquals($document1['body']['$collection'], $actors['body']['$id']); + $this->assertIsArray($document1['body']['$permissions']); + $this->assertIsArray($document1['body']['$permissions']['read']); + $this->assertIsArray($document1['body']['$permissions']['write']); + $this->assertCount(1, $document1['body']['$permissions']['read']); + $this->assertCount(1, $document1['body']['$permissions']['write']); + $this->assertEquals($document1['body']['firstName'], 'Tom'); + $this->assertEquals($document1['body']['lastName'], 'Holland'); + + $this->assertEquals($document2['headers']['status-code'], 201); + $this->assertEquals($document2['body']['$collection'], $actors['body']['$id']); + $this->assertIsArray($document2['body']['$permissions']); + $this->assertIsArray($document2['body']['$permissions']['read']); + $this->assertIsArray($document2['body']['$permissions']['write']); + $this->assertCount(1, $document2['body']['$permissions']['read']); + $this->assertCount(1, $document2['body']['$permissions']['write']); + $this->assertEquals($document2['body']['firstName'], 'Samuel'); + $this->assertEquals($document2['body']['lastName'], 'Jackson'); + + // Delete the actors collection + $response = $this->client->call(Client::METHOD_DELETE, '/database/collections/'.$actors['body']['$id'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], $this->getHeaders())); + + $this->assertEquals($response['headers']['status-code'], 204); + $this->assertEquals($response['body'],""); + + // Try to get the collection and check if it has been deleted + $response = $this->client->call(Client::METHOD_GET, '/database/collections/'.$actors['body']['$id'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders())); + + $this->assertEquals($response['headers']['status-code'], 404); + } } \ No newline at end of file