diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 39300a2d4a..92d3103293 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -205,81 +205,83 @@ App::post('/v1/projects') $sharedTablesV2 = !$projectTables && !$sharedTablesV1; $sharedTables = $sharedTablesV1 || $sharedTablesV2; - if ($sharedTables) { - $dbForProject - ->setSharedTables(true) - ->setTenant($sharedTablesV1 ? $project->getInternalId() : null) - ->setNamespace($dsn->getParam('namespace')); - } else { - $dbForProject - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } + if (!$sharedTablesV2) { + if ($sharedTables) { + $dbForProject + ->setSharedTables(true) + ->setTenant($sharedTablesV1 ? $project->getInternalId() : null) + ->setNamespace($dsn->getParam('namespace')); + } else { + $dbForProject + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } - $create = true; + $create = true; - try { - $dbForProject->create(); - } catch (Duplicate) { - $create = false; - } + try { + $dbForProject->create(); + } catch (Duplicate) { + $create = false; + } - if ($create || $projectTables) { - $audit = new Audit($dbForProject); - $audit->setup(); + if ($create || $projectTables) { + $audit = new Audit($dbForProject); + $audit->setup(); - $abuse = new TimeLimit('', 0, 1, $dbForProject); - $abuse->setup(); - } + $abuse = new TimeLimit('', 0, 1, $dbForProject); + $abuse->setup(); + } - if (!$create && $sharedTablesV1) { - $attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES); - $indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES); - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom('audit'), - '$permissions' => [Permission::create(Role::any())], - 'name' => 'audit', - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); + if (!$create && $sharedTablesV1) { + $attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES); + $indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES); + $dbForProject->createDocument(Database::METADATA, new Document([ + '$id' => ID::custom('audit'), + '$permissions' => [Permission::create(Role::any())], + 'name' => 'audit', + 'attributes' => $attributes, + 'indexes' => $indexes, + 'documentSecurity' => true + ])); - $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES); - $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES); - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom('abuse'), - '$permissions' => [Permission::create(Role::any())], - 'name' => 'abuse', - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); - } + $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES); + $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES); + $dbForProject->createDocument(Database::METADATA, new Document([ + '$id' => ID::custom('abuse'), + '$permissions' => [Permission::create(Role::any())], + 'name' => 'abuse', + 'attributes' => $attributes, + 'indexes' => $indexes, + 'documentSecurity' => true + ])); + } - if ($create || $sharedTablesV1) { - /** @var array $collections */ - $collections = Config::getParam('collections', [])['projects'] ?? []; + if ($create || $sharedTablesV1) { + /** @var array $collections */ + $collections = Config::getParam('collections', [])['projects'] ?? []; - foreach ($collections as $key => $collection) { - if (($collection['$collection'] ?? '') !== Database::METADATA) { - continue; - } + foreach ($collections as $key => $collection) { + if (($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } - $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); - $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); + $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); - try { - $dbForProject->createCollection($key, $attributes, $indexes); - } catch (Duplicate) { - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom($key), - '$permissions' => [Permission::create(Role::any())], - 'name' => $key, - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); + try { + $dbForProject->createCollection($key, $attributes, $indexes); + } catch (Duplicate) { + $dbForProject->createDocument(Database::METADATA, new Document([ + '$id' => ID::custom($key), + '$permissions' => [Permission::create(Role::any())], + 'name' => $key, + 'attributes' => $attributes, + 'indexes' => $indexes, + 'documentSecurity' => true + ])); + } } } } diff --git a/app/http.php b/app/http.php index 00c8eff21f..4bfacd4cac 100644 --- a/app/http.php +++ b/app/http.php @@ -12,6 +12,7 @@ use Swoole\Process; use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; @@ -90,7 +91,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg Console::success('[Setup] - Server database init started...'); try { - Console::success('[Setup] - Creating database: appwrite...'); + Console::success('[Setup] - Creating console database...'); $dbForConsole->create(); } catch (Duplicate) { Console::success('[Setup] - Skip: metadata table already exists'); @@ -117,34 +118,10 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg continue; } - Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...'); + Console::success('[Setup] - Creating console collection: ' . $collection['$id'] . '...'); - $attributes = []; - $indexes = []; - - foreach ($collection['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => ID::custom($attribute['$id']), - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } - - foreach ($collection['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => ID::custom($index['$id']), - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } + $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); $dbForConsole->createCollection($key, $attributes, $indexes); } @@ -179,36 +156,55 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg throw new Exception('Files collection is not configured.'); } - $attributes = []; - $indexes = []; - - foreach ($files['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => ID::custom($attribute['$id']), - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } - - foreach ($files['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => ID::custom($index['$id']), - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); - } + $attributes = \array_map(fn ($attribute) => new Document($attribute), $files['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $files['indexes']); $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } + $projectCollections = $collections['projects']; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); + $sharedTablesV2 = \array_diff($sharedTables, $sharedTablesV1); + + $cache = $app->getResource('cache'); + + foreach ($sharedTablesV2 as $hostname) { + $adapter = $pools + ->get($hostname) + ->pop() + ->getResource(); + + $dbForProject = (new Database($adapter, $cache)) + ->setDatabase('appwrite') + ->setSharedTables(true) + ->setTenant(null) + ->setNamespace(System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '')); + + try { + Console::success('[Setup] - Creating project database: ' . $hostname . '...'); + $dbForProject->create(); + } catch (Duplicate) { + Console::success('[Setup] - Skip: metadata table already exists'); + } + + foreach ($projectCollections as $key => $collection) { + if (($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } + if (!$dbForProject->getCollection($key)->isEmpty()) { + continue; + } + + $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); + + Console::success('[Setup] - Creating project collection: ' . $collection['$id'] . '...'); + + $dbForProject->createCollection($key, $attributes, $indexes); + } + } + $pools->reclaim(); Console::success('[Setup] - Server database init completed...'); diff --git a/app/init.php b/app/init.php index 3744bb055c..047e9402b6 100644 --- a/app/init.php +++ b/app/init.php @@ -1425,13 +1425,6 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); if (\in_array($dsn->getHost(), $sharedTables)) { diff --git a/docker-compose.yml b/docker-compose.yml index 0f3f05847c..bae2cc7811 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -194,6 +194,7 @@ services: - _APP_EXPERIMENT_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 + - _APP_DATABASE_SHARED_NAMESPACE appwrite-console: <<: *x-logging @@ -383,6 +384,7 @@ services: - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_DATABASE_SHARED_TABLES + - _APP_DATABASE_SHARED_TABLES_V1 appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index ddab8d0cf8..6c11116a4c 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -494,14 +494,21 @@ class Deletes extends Action ]; $limit = \count($projectCollectionIds) + 25; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); + + $projectTables = !\in_array($dsn->getHost(), $sharedTables); + $sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1); + $sharedTablesV2 = !$projectTables && !$sharedTablesV1; + $sharedTables = $sharedTablesV1 || $sharedTablesV2; while (true) { $collections = $dbForProject->listCollections($limit); foreach ($collections as $collection) { try { - if (!\in_array($dsn->getHost(), $sharedTables) || !\in_array($collection->getId(), $projectCollectionIds)) { + if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); } else { $this->deleteByGroup($collection->getId(), [], database: $dbForProject); @@ -511,7 +518,7 @@ class Deletes extends Action } } - if (\in_array($dsn->getHost(), $sharedTables)) { + if ($sharedTables) { $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections); if (empty(\array_diff($collectionsIds, $projectCollectionIds))) { @@ -565,10 +572,17 @@ class Deletes extends Action ], $dbForConsole); // Delete metadata table - if (!\in_array($dsn->getHost(), $sharedTables)) { + if ($projectTables) { $dbForProject->deleteCollection(Database::METADATA); - } else { + } elseif ($sharedTablesV1) { $this->deleteByGroup(Database::METADATA, [], $dbForProject); + } elseif ($sharedTablesV2) { + $queries = \array_map( + fn ($id) => Query::notEqual('$id', $id), + $projectCollectionIds + ); + + $this->deleteByGroup(Database::METADATA, $queries, $dbForProject); } // Delete all storage directories diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index e3f48bb2a6..326443afcf 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -3889,6 +3889,18 @@ class ProjectsConsoleClientTest extends Scope ]); $this->assertEquals(200, $user2['headers']['status-code']); + + // Create another user in project 2 in case read hits stale cache + $user3 = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2Id, + 'x-appwrite-key' => $key2['body']['secret'], + ], [ + 'userId' => ID::unique(), + 'email' => 'test3@appwrite.io' + ]); + + $this->assertEquals(201, $user3['headers']['status-code']); } /**