Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-24 09:28:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Fix deletes worker not deleting project database tables

Browse source
This commit is contained in:
Jake Barnby 2023-01-11 19:37:06 +13:00
parent 514b42bea2
commit 4b0ef4598b
No known key found for this signature in database
GPG key ID: C437A8CC85B96E9C
2 changed files with 105 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
app/workers/deletes.php
View file

@ -172,6 +172,7 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document database document * @param Document $document database document
* @param string $projectId * @param string $projectId
* @throws Exception
*/ */
protected function deleteDatabase(Document $document, string $projectId): void protected function deleteDatabase(Document $document, string $projectId): void
{ {
@ -191,6 +192,7 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document teams document * @param Document $document teams document
* @param string $projectId * @param string $projectId
* @throws Exception
*/ */
protected function deleteCollection(Document $document, string $projectId): void protected function deleteCollection(Document $document, string $projectId): void
{ {
@ -217,6 +219,7 @@ class DeletesV1 extends Worker
/** /**
* @param string $hourlyUsageRetentionDatetime * @param string $hourlyUsageRetentionDatetime
* @throws Exception
*/ */
protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) protected function deleteUsageStats(string $hourlyUsageRetentionDatetime)
{ {
@ -232,6 +235,7 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document teams document * @param Document $document teams document
* @param string $projectId * @param string $projectId
* @throws Exception
*/ */
protected function deleteMemberships(Document $document, string $projectId): void protected function deleteMemberships(Document $document, string $projectId): void
{ {
@ -245,13 +249,37 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document project document * @param Document $document project document
* @throws Exception
*/ */
protected function deleteProject(Document $document): void protected function deleteProject(Document $document): void
{ {
$projectId = $document->getId(); // Delete project tables
$dbForProject = $this->getProjectDBFromDocument($document);
// Delete all DBs $limit = 50;
$this->getProjectDB($projectId)->delete($projectId); $offset = 0;
while (true) {
$collections = $dbForProject->listCollections($limit, $offset);
if (empty($collections)) {
break;
}
foreach ($collections as $collection) {
$dbForProject->deleteCollection($collection->getId());
}
$offset += $limit;
}
// Delete metadata tables
try {
$dbForProject->deleteCollection('_metadata');
} catch (Exception) {
// Ignore: deleteCollection tries to delete a metadata entry after the collection is deleted,
// which will throw an exception here because the metadata collection is already deleted.
}
// Delete all storage directories // Delete all storage directories
$uploads = $this->getFilesDevice($document->getId()); $uploads = $this->getFilesDevice($document->getId());
@ -264,6 +292,7 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document user document * @param Document $document user document
* @param string $projectId * @param string $projectId
* @throws Exception
*/ */
protected function deleteUser(Document $document, string $projectId): void protected function deleteUser(Document $document, string $projectId): void
{ {
@ -305,6 +334,7 @@ class DeletesV1 extends Worker
/** /**
* @param string $datetime * @param string $datetime
* @throws Exception
*/ */
protected function deleteExecutionLogs(string $datetime): void protected function deleteExecutionLogs(string $datetime): void
{ {
@ -337,6 +367,7 @@ class DeletesV1 extends Worker
/** /**
* @param string $datetime * @param string $datetime
* @throws Exception
*/ */
protected function deleteRealtimeUsage(string $datetime): void protected function deleteRealtimeUsage(string $datetime): void
{ {
@ -393,6 +424,7 @@ class DeletesV1 extends Worker
/** /**
* @param string $resource * @param string $resource
* @param string $projectId * @param string $projectId
* @throws Exception
*/ */
protected function deleteAuditLogsByResource(string $resource, string $projectId): void protected function deleteAuditLogsByResource(string $resource, string $projectId): void
{ {
@ -406,6 +438,7 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document function document * @param Document $document function document
* @param string $projectId * @param string $projectId
* @throws Exception
*/ */
protected function deleteFunction(Document $document, string $projectId): void protected function deleteFunction(Document $document, string $projectId): void
{ {
@ -479,6 +512,7 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document deployment document * @param Document $document deployment document
* @param string $projectId * @param string $projectId
* @throws Exception
*/ */
protected function deleteDeployment(Document $document, string $projectId): void protected function deleteDeployment(Document $document, string $projectId): void
{ {
@ -528,9 +562,10 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document to be deleted * @param Document $document to be deleted
* @param Database $database to delete it from * @param Database $database to delete it from
* @param callable $callback to perform after document is deleted * @param callable|null $callback to perform after document is deleted
* *
* @return bool * @return bool
* @throws \Utopia\Database\Exception\Authorization
*/ */
protected function deleteById(Document $document, Database $database, callable $callback = null): bool protected function deleteById(Document $document, Database $database, callable $callback = null): bool
{ {
@ -550,6 +585,7 @@ class DeletesV1 extends Worker
/** /**
* @param callable $callback * @param callable $callback
* @throws Exception
*/ */
protected function deleteForProjectIds(callable $callback): void protected function deleteForProjectIds(callable $callback): void
{ {
@ -584,9 +620,10 @@ class DeletesV1 extends Worker
/** /**
* @param string $collection collectionID * @param string $collection collectionID
* @param Query[] $queries * @param array $queries
* @param Database $database * @param Database $database
* @param callable $callback * @param callable|null $callback
* @throws Exception
*/ */
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
{ {
@ -620,6 +657,7 @@ class DeletesV1 extends Worker
/** /**
* @param Document $document certificates document * @param Document $document certificates document
* @throws \Utopia\Database\Exception\Authorization
*/ */
protected function deleteCertificates(Document $document): void protected function deleteCertificates(Document $document): void
{ {

92
src/Appwrite/Resque/Worker.php
View file

@ -2,22 +2,23 @@
namespace Appwrite\Resque; namespace Appwrite\Resque;
use Exception;
use Utopia\App; use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Cache\Cache;
use Utopia\CLI\Console; use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Storage\Device; use Utopia\Storage\Device;
use Utopia\Storage\Storage; use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\S3; use Utopia\Storage\Device\S3;
use Exception; use Utopia\Storage\Device\Wasabi;
use Utopia\Database\Validator\Authorization; use Utopia\Storage\Storage;
abstract class Worker abstract class Worker
{ {
@ -53,7 +54,7 @@ abstract class Worker
* @return void * @return void
* @throws \Exception|\Throwable * @throws \Exception|\Throwable
*/ */
public function init() public function init(): void
{ {
throw new Exception("Please implement init method in worker"); throw new Exception("Please implement init method in worker");
} }
@ -65,7 +66,7 @@ abstract class Worker
* @return void * @return void
* @throws \Exception|\Throwable * @throws \Exception|\Throwable
*/ */
public function run() public function run(): void
{ {
throw new Exception("Please implement run method in worker"); throw new Exception("Please implement run method in worker");
} }
@ -77,7 +78,7 @@ abstract class Worker
* @return void * @return void
* @throws \Exception|\Throwable * @throws \Exception|\Throwable
*/ */
public function shutdown() public function shutdown(): void
{ {
throw new Exception("Please implement shutdown method in worker"); throw new Exception("Please implement shutdown method in worker");
} }
@ -151,17 +152,18 @@ abstract class Worker
/** /**
* Register callback. Will be executed when error occurs. * Register callback. Will be executed when error occurs.
* @param callable $callback * @param callable $callback
* @param Throwable $error * @return void
* @return self
*/ */
public static function error(callable $callback): void public static function error(callable $callback): void
{ {
\array_push(self::$errorCallbacks, $callback); self::$errorCallbacks[] = $callback;
} }
/** /**
* Get internal project database * Get internal project database
* @param string $projectId * @param string $projectId
* @return Database * @return Database
* @throws Exception
*/ */
protected function getProjectDB(string $projectId): Database protected function getProjectDB(string $projectId): Database
{ {
@ -177,9 +179,23 @@ abstract class Worker
return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId()); return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId());
} }
/**
* Get internal project database given the project document
*
* Allows avoiding race conditions when modifying the projects collection
* @param Document $project
* @return Database
* @throws Exception
*/
protected function getProjectDBFromDocument(Document $project): Database
{
return $this->getDB(self::DATABASE_PROJECT, project: $project);
}
/** /**
* Get console database * Get console database
* @return Database * @return Database
* @throws Exception
*/ */
protected function getConsoleDB(): Database protected function getConsoleDB(): Database
{ {
@ -187,24 +203,35 @@ abstract class Worker
} }
/** /**
* Get console database * Get database
* @param string $type One of (internal, external, console) * @param string $type One of (project, console)
* @param string $projectId of internal or external DB * @param string $projectId of project or console DB
* @param string $projectInternalId
* @param Document|null $project
* @return Database * @return Database
* @throws Exception
*/ */
private function getDB(string $type, string $projectId = '', string $projectInternalId = ''): Database private function getDB(
{ string $type,
string $projectId = '',
string $projectInternalId = '',
?Document $project = null
): Database {
global $register; global $register;
$namespace = '';
$sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary $sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary
if ($project !== null) {
$projectId = $project->getId();
$projectInternalId = $project->getInternalId();
}
switch ($type) { switch ($type) {
case self::DATABASE_PROJECT: case self::DATABASE_PROJECT:
if (!$projectId) { if (!$projectId) {
throw new \Exception('ProjectID not provided - cannot get database'); throw new \Exception('ProjectID not provided - cannot get database');
} }
$namespace = "_{$projectInternalId}"; $namespace = "_$projectInternalId";
break; break;
case self::DATABASE_CONSOLE: case self::DATABASE_CONSOLE:
$namespace = "_console"; $namespace = "_console";
@ -212,12 +239,11 @@ abstract class Worker
break; break;
default: default:
throw new \Exception('Unknown database type: ' . $type); throw new \Exception('Unknown database type: ' . $type);
break;
} }
$attempts = 0; $attempts = 0;
do { while (true) {
try { try {
$attempts++; $attempts++;
$cache = new Cache(new RedisCache($register->get('cache'))); $cache = new Cache(new RedisCache($register->get('cache')));
@ -225,8 +251,12 @@ abstract class Worker
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace($namespace); // Main DB $database->setNamespace($namespace); // Main DB
if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) { if (
throw new \Exception("Project does not exist: {$projectId}"); $project === null
&& !empty($projectId)
&& !$database->getDocument('projects', $projectId)->isEmpty()
) {
throw new \Exception("Project does not exist: $projectId");
} }
if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) { if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) {
@ -235,13 +265,13 @@ abstract class Worker
break; // leave loop if successful break; // leave loop if successful
} catch (\Exception $e) { } catch (\Exception $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})..."); Console::warning("Database not ready. Retrying connection ($attempts)...");
if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage()); throw new \Exception('Failed to connect to database: ' . $e->getMessage());
} }
sleep($sleep); sleep($sleep);
} }
} while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); }
return $database; return $database;
} }
@ -251,7 +281,7 @@ abstract class Worker
* @param string $projectId of the project * @param string $projectId of the project
* @return Device * @return Device
*/ */
protected function getFunctionsDevice($projectId): Device protected function getFunctionsDevice(string $projectId): Device
{ {
return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
} }
@ -261,7 +291,7 @@ abstract class Worker
* @param string $projectId of the project * @param string $projectId of the project
* @return Device * @return Device
*/ */
protected function getFilesDevice($projectId): Device protected function getFilesDevice(string $projectId): Device
{ {
return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
} }
@ -272,7 +302,7 @@ abstract class Worker
* @param string $projectId of the project * @param string $projectId of the project
* @return Device * @return Device
*/ */
protected function getBuildsDevice($projectId): Device protected function getBuildsDevice(string $projectId): Device
{ {
return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId);
} }
@ -282,7 +312,7 @@ abstract class Worker
* @param string $root path of the device * @param string $root path of the device
* @return Device * @return Device
*/ */
public function getDevice($root): Device public function getDevice(string $root): Device
{ {
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL: case Storage::DEVICE_LOCAL: