Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-23 00:49:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #9013 from appwrite/feat-db-create-on-server-start

Browse source 
Improved shared tables V2
This commit is contained in:
Christy Jacob 2024-11-21 10:41:52 +01:00 committed by GitHub
commit 34d8a0504b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 149 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
app/controllers/api/projects.php
View file

@ -205,81 +205,83 @@ App::post('/v1/projects')
$sharedTablesV2 = !$projectTables && !$sharedTablesV1;
$sharedTables = $sharedTablesV1 || $sharedTablesV2;
if ($sharedTables) {
$dbForProject
->setSharedTables(true)
->setTenant($sharedTablesV1 ? $project->getInternalId() : null)
->setNamespace($dsn->getParam('namespace'));
} else {
$dbForProject
->setSharedTables(false)
->setTenant(null)
->setNamespace('_' . $project->getInternalId());
}
if (!$sharedTablesV2) {
if ($sharedTables) {
$dbForProject
->setSharedTables(true)
->setTenant($sharedTablesV1 ? $project->getInternalId() : null)
->setNamespace($dsn->getParam('namespace'));
} else {
$dbForProject
->setSharedTables(false)
->setTenant(null)
->setNamespace('_' . $project->getInternalId());
}
$create = true;
$create = true;
try {
$dbForProject->create();
} catch (Duplicate) {
$create = false;
}
try {
$dbForProject->create();
} catch (Duplicate) {
$create = false;
}
if ($create || $projectTables) {
$audit = new Audit($dbForProject);
$audit->setup();
if ($create || $projectTables) {
$audit = new Audit($dbForProject);
$audit->setup();
$abuse = new TimeLimit('', 0, 1, $dbForProject);
$abuse->setup();
}
$abuse = new TimeLimit('', 0, 1, $dbForProject);
$abuse->setup();
}
if (!$create && $sharedTablesV1) {
$attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('audit'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'audit',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
if (!$create && $sharedTablesV1) {
$attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('audit'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'audit',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
$attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('abuse'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'abuse',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('abuse'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'abuse',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
}
if ($create || $sharedTablesV1) {
/** @var array $collections */
$collections = Config::getParam('collections', [])['projects'] ?? [];
if ($create || $sharedTablesV1) {
/** @var array $collections */
$collections = Config::getParam('collections', [])['projects'] ?? [];
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
try {
$dbForProject->createCollection($key, $attributes, $indexes);
} catch (Duplicate) {
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom($key),
'$permissions' => [Permission::create(Role::any())],
'name' => $key,
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
try {
$dbForProject->createCollection($key, $attributes, $indexes);
} catch (Duplicate) {
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom($key),
'$permissions' => [Permission::create(Role::any())],
'name' => $key,
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
}
}
}
}

104
app/http.php
View file

@ -12,6 +12,7 @@ use Swoole\Process;
use Utopia\Abuse\Adapters\Database\TimeLimit;
use Utopia\App;
use Utopia\Audit\Audit;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
@ -90,7 +91,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
Console::success('[Setup] - Server database init started...');
try {
Console::success('[Setup] - Creating database: appwrite...');
Console::success('[Setup] - Creating console database...');
$dbForConsole->create();
} catch (Duplicate) {
Console::success('[Setup] - Skip: metadata table already exists');
@ -117,34 +118,10 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
continue;
}
Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...');
Console::success('[Setup] - Creating console collection: ' . $collection['$id'] . '...');
$attributes = [];
$indexes = [];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => ID::custom($attribute['$id']),
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
'default' => $attribute['default'] ?? null,
'format' => $attribute['format'] ?? ''
]);
}
foreach ($collection['indexes'] as $index) {
$indexes[] = new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
$dbForConsole->createCollection($key, $attributes, $indexes);
}
@ -179,36 +156,55 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
throw new Exception('Files collection is not configured.');
}
$attributes = [];
$indexes = [];
foreach ($files['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => ID::custom($attribute['$id']),
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
'default' => $attribute['default'] ?? null,
'format' => $attribute['format'] ?? ''
]);
}
foreach ($files['indexes'] as $index) {
$indexes[] = new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $files['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $files['indexes']);
$dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
}
$projectCollections = $collections['projects'];
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
$sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
$sharedTablesV2 = \array_diff($sharedTables, $sharedTablesV1);
$cache = $app->getResource('cache');
foreach ($sharedTablesV2 as $hostname) {
$adapter = $pools
->get($hostname)
->pop()
->getResource();
$dbForProject = (new Database($adapter, $cache))
->setDatabase('appwrite')
->setSharedTables(true)
->setTenant(null)
->setNamespace(System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''));
try {
Console::success('[Setup] - Creating project database: ' . $hostname . '...');
$dbForProject->create();
} catch (Duplicate) {
Console::success('[Setup] - Skip: metadata table already exists');
}
foreach ($projectCollections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$dbForProject->getCollection($key)->isEmpty()) {
continue;
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
Console::success('[Setup] - Creating project collection: ' . $collection['$id'] . '...');
$dbForProject->createCollection($key, $attributes, $indexes);
}
}
$pools->reclaim();
Console::success('[Setup] - Server database init completed...');

7
app/init.php
View file

@ -1425,13 +1425,6 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
try {
$dsn = new DSN($project->getAttribute('database'));
} catch (\InvalidArgumentException) {
// TODO: Temporary until all projects are using shared tables
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {

2
docker-compose.yml
View file

@ -194,6 +194,7 @@ services:
- _APP_EXPERIMENT_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
- _APP_DATABASE_SHARED_TABLES_V1
- _APP_DATABASE_SHARED_NAMESPACE
appwrite-console:
<<: *x-logging
@ -383,6 +384,7 @@ services:
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_DATABASE_SHARED_TABLES
- _APP_DATABASE_SHARED_TABLES_V1
appwrite-worker-databases:
entrypoint: worker-databases

22
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -494,14 +494,21 @@ class Deletes extends Action
];
$limit = \count($projectCollectionIds) + 25;
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
$sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
$projectTables = !\in_array($dsn->getHost(), $sharedTables);
$sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1);
$sharedTablesV2 = !$projectTables && !$sharedTablesV1;
$sharedTables = $sharedTablesV1 || $sharedTablesV2;
while (true) {
$collections = $dbForProject->listCollections($limit);
foreach ($collections as $collection) {
try {
if (!\in_array($dsn->getHost(), $sharedTables) || !\in_array($collection->getId(), $projectCollectionIds)) {
if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) {
$dbForProject->deleteCollection($collection->getId());
} else {
$this->deleteByGroup($collection->getId(), [], database: $dbForProject);
@ -511,7 +518,7 @@ class Deletes extends Action
}
}
if (\in_array($dsn->getHost(), $sharedTables)) {
if ($sharedTables) {
$collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections);
if (empty(\array_diff($collectionsIds, $projectCollectionIds))) {
@ -565,10 +572,17 @@ class Deletes extends Action
], $dbForConsole);
// Delete metadata table
if (!\in_array($dsn->getHost(), $sharedTables)) {
if ($projectTables) {
$dbForProject->deleteCollection(Database::METADATA);
} else {
} elseif ($sharedTablesV1) {
$this->deleteByGroup(Database::METADATA, [], $dbForProject);
} elseif ($sharedTablesV2) {
$queries = \array_map(
fn ($id) => Query::notEqual('$id', $id),
$projectCollectionIds
);
$this->deleteByGroup(Database::METADATA, $queries, $dbForProject);
}
// Delete all storage directories

12
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
View file

@ -3889,6 +3889,18 @@ class ProjectsConsoleClientTest extends Scope
]);
$this->assertEquals(200, $user2['headers']['status-code']);
// Create another user in project 2 in case read hits stale cache
$user3 = $this->client->call(Client::METHOD_POST, '/users', [
'content-type' => 'application/json',
'x-appwrite-project' => $project2Id,
'x-appwrite-key' => $key2['body']['secret'],
], [
'userId' => ID::unique(),
'email' => 'test3@appwrite.io'
]);
$this->assertEquals(201, $user3['headers']['status-code']);
}
/**