diff --git a/.env b/.env index 09abb07be2..b889ebb513 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ _APP_ENV=development +_APP_EDITION=self-hosted _APP_LOCALE=en _APP_WORKER_PER_CORE=6 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/.gitignore b/.gitignore index ac88830b49..ef05c948ff 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ app/sdks dev/yasd_init.php .phpunit.result.cache Makefile +appwrite.json diff --git a/app/cli.php b/app/cli.php index 1996b4da32..da7d23c18d 100644 --- a/app/cli.php +++ b/app/cli.php @@ -15,6 +15,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; +use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Platform\Service; use Utopia\Pools\Group; @@ -98,25 +99,53 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $dbForConsole; } - $databaseName = $project->getAttribute('database'); + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + if (isset($databases[$dsn->getHost()])) { + $database = $databases[$dsn->getHost()]; + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } - if (isset($databases[$databaseName])) { - $database = $databases[$databaseName]; - $database->setNamespace('_' . $project->getInternalId()); return $database; } $dbAdapter = $pools - ->get($databaseName) + ->get($dsn->getHost()) ->pop() ->getResource(); $database = new Database($dbAdapter, $cache); - $databases[$databaseName] = $database; + $databases[$dsn->getHost()] = $database; + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } $database - ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()); diff --git a/app/config/collections.php b/app/config/collections.php index cd0d1b2ef5..72d126e343 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -390,7 +390,7 @@ $commonCollections = [ '$id' => ID::custom('_key_email'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['email'], - 'lengths' => [320], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ @@ -996,7 +996,7 @@ $commonCollections = [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_KEY, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [100, 100], + 'lengths' => [128, 128], 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ @@ -1120,14 +1120,14 @@ $commonCollections = [ '$id' => ID::custom('_key_userInternalId_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['userInternalId', 'provider', 'providerUid'], - 'lengths' => [Database::LENGTH_KEY, 100, 385], + 'lengths' => [11, 128, 128], 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [100, 640], + 'lengths' => [128, 128], 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ @@ -1148,7 +1148,7 @@ $commonCollections = [ '$id' => ID::custom('_key_provider'), 'type' => Database::INDEX_KEY, 'attributes' => ['provider'], - 'lengths' => [100], + 'lengths' => [128], 'orders' => [Database::ORDER_ASC], ], [ @@ -3068,7 +3068,7 @@ $projectCollections = array_merge([ '$id' => ID::custom('_key_name'), 'type' => Database::INDEX_KEY, 'attributes' => ['name'], - 'lengths' => [768], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ @@ -3117,7 +3117,7 @@ $projectCollections = array_merge([ '$id' => ID::custom('_key_runtime'), 'type' => Database::INDEX_KEY, 'attributes' => ['runtime'], - 'lengths' => [768], + 'lengths' => [64], 'orders' => [Database::ORDER_ASC], ], [ @@ -3857,14 +3857,14 @@ $projectCollections = array_merge([ '$id' => ID::custom('_key_trigger'), 'type' => Database::INDEX_KEY, 'attributes' => ['trigger'], - 'lengths' => [128], + 'lengths' => [32], 'orders' => [Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_status'), 'type' => Database::INDEX_KEY, 'attributes' => ['status'], - 'lengths' => [128], + 'lengths' => [32], 'orders' => [Database::ORDER_ASC], ], [ @@ -5833,14 +5833,14 @@ $bucketCollections = [ '$id' => ID::custom('_key_name'), 'type' => Database::INDEX_KEY, 'attributes' => ['name'], - 'lengths' => [768], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_signature'), 'type' => Database::INDEX_KEY, 'attributes' => ['signature'], - 'lengths' => [768], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index 23d62826da..337ba8fd9e 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -16,6 +16,7 @@ use Utopia\Domains\Domain; use Utopia\Fetch\Client; use Utopia\Image\Image; use Utopia\Logger\Logger; +use Utopia\System\System; use Utopia\Validator\Boolean; use Utopia\Validator\HexColor; use Utopia\Validator\Range; @@ -319,8 +320,8 @@ App::get('/v1/avatars/favicon') ->setAllowRedirects(false) ->setUserAgent(\sprintf( APP_USERAGENT, - App::getEnv('_APP_VERSION', 'UNKNOWN'), - App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY) + System::getEnv('_APP_VERSION', 'UNKNOWN'), + System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY) )) ->fetch($url); } catch (\Throwable) { diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 72f56c96e9..c1ffd6466f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -11,6 +11,7 @@ use Appwrite\Network\Validator\Origin; use Appwrite\Template\Template; use Appwrite\Utopia\Database\Validator\ProjectId; use Appwrite\Utopia\Database\Validator\Queries\Projects; +use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use PHPMailer\PHPMailer\PHPMailer; use Utopia\Abuse\Adapters\TimeLimit; @@ -29,6 +30,7 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; use Utopia\Database\Validator\UID; use Utopia\Domains\Validator\PublicDomain; +use Utopia\DSN\DSN; use Utopia\Locale\Locale; use Utopia\Pools\Group; use Utopia\System\System; @@ -74,12 +76,13 @@ App::post('/v1/projects') ->param('legalCity', '', new Text(256), 'Project legal City. Max length: 256 chars.', true) ->param('legalAddress', '', new Text(256), 'Project legal Address. Max length: 256 chars.', true) ->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true) + ->inject('request') ->inject('response') ->inject('dbForConsole') ->inject('cache') ->inject('pools') ->inject('hooks') - ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools, Hooks $hooks) { + ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Request $request, Response $response, Database $dbForConsole, Cache $cache, Group $pools, Hooks $hooks) { $team = $dbForConsole->getDocument('teams', $teamId); @@ -94,8 +97,15 @@ App::post('/v1/projects') } $auth = Config::getParam('auth', []); - $auths = ['limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false]; - foreach ($auth as $index => $method) { + $auths = [ + 'limit' => 0, + 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, + 'passwordHistory' => 0, + 'passwordDictionary' => false, + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, + 'personalDataCheck' => false + ]; + foreach ($auth as $method) { $auths[$method['key'] ?? ''] = true; } @@ -130,18 +140,50 @@ App::post('/v1/projects') } } - $databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE', null); - $index = array_search($databaseOverride, $databases); + $databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE'); + $index = \array_search($databaseOverride, $databases); if ($index !== false) { - $database = $databases[$index]; + $dsn = $databases[$index]; } else { - $database = $databases[array_rand($databases)]; + $dsn = $databases[array_rand($databases)]; } if ($projectId === 'console') { throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); } + // TODO: One in 20 projects use shared tables. Temporary until all projects are using shared tables. + if ( + !\mt_rand(0, 19) + && System::getEnv('_APP_DATABASE_SHARED_TABLES', 'enabled') === 'enabled' + && System::getEnv('_APP_EDITION', 'self-hosted') !== 'self-hosted' + ) { + $schema = 'appwrite'; + $database = 'appwrite'; + $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''); + $dsn = $schema . '://' . DATABASE_SHARED_TABLES . '?database=' . $database; + + if (!empty($namespace)) { + $dsn .= '&namespace=' . $namespace; + } + } + + // TODO: Allow overriding in development mode. Temporary until all projects are using shared tables. + if ( + App::isDevelopment() + && System::getEnv('_APP_EDITION', 'self-hosted') !== 'self-hosted' + && $request->getHeader('x-appwrited-share-tables', false) + ) { + $schema = 'appwrite'; + $database = 'appwrite'; + $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''); + $dsn = $schema . '://' . DATABASE_SHARED_TABLES . '?database=' . $database; + + if (!empty($namespace)) { + $dsn .= '&namespace=' . $namespace; + } + } + try { $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, @@ -173,21 +215,41 @@ App::post('/v1/projects') 'keys' => null, 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), - 'database' => $database + 'database' => $dsn, ])); - } catch (Duplicate $th) { + } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); } - $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); - $dbForProject->setNamespace("_{$project->getInternalId()}"); + try { + $dsn = new DSN($dsn); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $dsn); + } + + $adapter = $pools->get($dsn->getHost())->pop()->getResource(); + $dbForProject = new Database($adapter, $cache); + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $dbForProject + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $dbForProject + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + $dbForProject->create(); $audit = new Audit($dbForProject); $audit->setup(); - $adapter = new TimeLimit('', 0, 1, $dbForProject); - $adapter->setup(); + $abuse = new TimeLimit('', 0, 1, $dbForProject); + $abuse->setup(); /** @var array $collections */ $collections = Config::getParam('collections', [])['projects'] ?? []; @@ -197,33 +259,19 @@ App::post('/v1/projects') continue; } - $attributes = []; - $indexes = []; + $attributes = \array_map(function (array $attribute) { + return new Document($attribute); + }, $collection['attributes']); - foreach ($collection['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => $attribute['$id'], - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } + $indexes = \array_map(function (array $index) { + return new Document($index); + }, $collection['indexes']); - foreach ($collection['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => $index['$id'], - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); + try { + $dbForProject->createCollection($key, $attributes, $indexes); + } catch (Duplicate) { + // Collection already exists } - $dbForProject->createCollection($key, $attributes, $indexes); } $hooks->trigger('afterProjectCreation', [ $project, $pools, $cache ]); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 37c5c032e1..9215d99345 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -63,7 +63,7 @@ App::post('/v1/storage/buckets') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) diff --git a/app/controllers/general.php b/app/controllers/general.php index f93289a49b..bcdf0fecb3 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -419,7 +419,9 @@ App::init() Request::setRoute($route); if ($route === null) { - return $response->setStatusCode(404)->send('Not Found'); + return $response + ->setStatusCode(404) + ->send('Not Found'); } $requestFormat = $request->getHeader('x-appwrite-response-format', System::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', '')); @@ -572,7 +574,7 @@ App::init() ->addHeader('Server', 'Appwrite') ->addHeader('X-Content-Type-Options', 'nosniff') ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE') - ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Forwarded-For, X-Forwarded-User-Agent') + ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Shared-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Forwarded-For, X-Forwarded-User-Agent') ->addHeader('Access-Control-Expose-Headers', 'X-Appwrite-Session, X-Fallback-Cookies') ->addHeader('Access-Control-Allow-Origin', $refDomain) ->addHeader('Access-Control-Allow-Credentials', 'true'); @@ -623,7 +625,7 @@ App::options() $response ->addHeader('Server', 'Appwrite') ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE') - ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent') + ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Shared-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent') ->addHeader('Access-Control-Expose-Headers', 'X-Appwrite-Session, X-Fallback-Cookies') ->addHeader('Access-Control-Allow-Origin', $origin) ->addHeader('Access-Control-Allow-Credentials', 'true') @@ -711,8 +713,8 @@ App::error() if ($error->getCode() >= 400 && $error->getCode() < 500) { // Register error logger - $providerName = App::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', ''); - $providerConfig = App::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', ''); + $providerName = System::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', ''); + $providerConfig = System::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', ''); if (!(empty($providerName) || empty($providerConfig))) { if (!Logger::hasProvider($providerName)) { diff --git a/app/init.php b/app/init.php index c073f48a0e..7003b5c0fa 100644 --- a/app/init.php +++ b/app/init.php @@ -12,11 +12,11 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { require_once __DIR__ . '/../vendor/autoload.php'; } -ini_set('memory_limit', '512M'); -ini_set('display_errors', 1); -ini_set('display_startup_errors', 1); -ini_set('default_socket_timeout', -1); -error_reporting(E_ALL); +\ini_set('memory_limit', '512M'); +\ini_set('display_errors', 1); +\ini_set('display_startup_errors', 1); +\ini_set('default_socket_timeout', -1); +\error_reporting(E_ALL); use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; @@ -143,6 +143,9 @@ const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; const APP_HOSTNAME_INTERNAL = 'appwrite'; +// Databases +const DATABASE_SHARED_TABLES = 'database_db_fra1_self_hosted_16_0'; + // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; @@ -807,14 +810,13 @@ $register->set('pools', function () { foreach ($connections as $key => $connection) { $type = $connection['type'] ?? ''; - $dsns = $connection['dsns'] ?? ''; - $multipe = $connection['multiple'] ?? false; + $multiple = $connection['multiple'] ?? false; $schemes = $connection['schemes'] ?? []; $config = []; $dsns = explode(',', $connection['dsns'] ?? ''); foreach ($dsns as &$dsn) { $dsn = explode('=', $dsn); - $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $name = ($multiple) ? $key . '_' . $dsn[0] : $key; $dsn = $dsn[1] ?? ''; $config[] = $name; if (empty($dsn)) { @@ -837,42 +839,35 @@ $register->set('pools', function () { /** * Get Resource * - * Creation could be reused accross connection types like database, cache, queue, etc. + * Creation could be reused across connection types like database, cache, queue, etc. * * Resource assignment to an adapter will happen below. */ - switch ($dsnScheme) { - case 'mysql': - case 'mariadb': - $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( - // No need to set PDO::ATTR_ERRMODE it is overwitten in PDOProxy - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - )); - }); - }; - break; - case 'redis': - $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); - @$redis->pconnect($dsnHost, (int)$dsnPort); - if ($dsnPass) { - $redis->auth($dsnPass); - } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + $resource = match ($dsnScheme) { + 'mysql', + 'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }, + 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - return $redis; - }; - break; - - default: - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); - } + return $redis; + }, + default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'), + }; $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { // Get Adapter @@ -1096,24 +1091,25 @@ App::setResource('clients', function ($request, $console, $project) { fn ($node) => $node['hostname'], \array_filter( $console->getAttribute('platforms', []), - fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && isset($node['hostname']) && !empty($node['hostname'])) + fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && !empty($node['hostname'])) ) ); - $clients = \array_unique( - \array_merge( - $clientsConsole, - \array_map( - fn ($node) => $node['hostname'], - \array_filter( - $project->getAttribute('platforms', []), - fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB || $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && isset($node['hostname']) && !empty($node['hostname'])) - ) - ) - ) - ); + $clients = $clientsConsole; + $platforms = $project->getAttribute('platforms', []); - return $clients; + foreach ($platforms as $node) { + if ( + isset($node['type']) && + ($node['type'] === Origin::CLIENT_TYPE_WEB || + $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && + !empty($node['hostname']) + ) { + $clients[] = $node['hostname']; + } + } + + return \array_unique($clients); }, ['request', 'console', 'project']); App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForConsole) { @@ -1306,19 +1302,44 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, return $dbForConsole; } + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + $dbAdapter = $pools - ->get($project->getAttribute('database')) + ->get($dsn->getHost()) ->pop() ->getResource(); $database = new Database($dbAdapter, $cache); $database - ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + return $database; }, ['pools', 'dbForConsole', 'cache', 'project']); @@ -1326,8 +1347,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $dbAdapter = $pools ->get('console') ->pop() - ->getResource() - ; + ->getResource(); $database = new Database($dbAdapter, $cache); @@ -1343,44 +1363,54 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { + return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } - $databaseName = $project->getAttribute('database'); - - if (isset($databases[$databaseName])) { - $database = $databases[$databaseName]; + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + $configure = (function (Database $database) use ($project, $dsn) { $database - ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + }); + + if (isset($databases[$dsn->getHost()])) { + $database = $databases[$dsn->getHost()]; + $configure($database); return $database; } $dbAdapter = $pools - ->get($databaseName) + ->get($dsn->getHost()) ->pop() ->getResource(); $database = new Database($dbAdapter, $cache); - - $databases[$databaseName] = $database; - - $database - ->setNamespace('_' . $project->getInternalId()) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + $databases[$dsn->getHost()] = $database; + $configure($database); return $database; }; - - return $getProjectDB; }, ['pools', 'dbForConsole', 'cache']); App::setResource('cache', function (Group $pools) { diff --git a/app/realtime.php b/app/realtime.php index ce84539251..edf83a5694 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -26,6 +26,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\System\System; use Utopia\WebSocket\Adapter; @@ -77,16 +78,33 @@ if (!function_exists("getProjectDB")) { return getConsoleDB(); } - $dbAdapter = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } - $database = new Database($dbAdapter, getCache()); + $adapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($adapter, getCache()); + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } $database - ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()); diff --git a/app/worker.php b/app/worker.php index 6f912a84fd..d698a824ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,6 +16,7 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; +use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -24,6 +25,7 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; +use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Platform\Service; @@ -70,14 +72,41 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, } $pools = $register->get('pools'); - $database = $pools - ->get($project->getAttribute('database')) + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $adapter = $pools + ->get($dsn->getHost()) ->pop() ->getResource(); - $adapter = new Database($database, $cache); - $adapter->setNamespace('_' . $project->getInternalId()); - return $adapter; + $database = new Database($adapter, $cache); + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; }, ['cache', 'register', 'message', 'project', 'dbForConsole']); Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { @@ -88,24 +117,51 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso return $dbForConsole; } - $databaseName = $project->getAttribute('database'); + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + if (isset($databases[$dsn->getHost()])) { + $database = $databases[$dsn->getHost()]; + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } - if (isset($databases[$databaseName])) { - $database = $databases[$databaseName]; - $database->setNamespace('_' . $project->getInternalId()); return $database; } $dbAdapter = $pools - ->get($databaseName) + ->get($dsn->getHost()) ->pop() ->getResource(); $database = new Database($dbAdapter, $cache); - $databases[$databaseName] = $database; + $databases[$dsn->getHost()] = $database; - $database->setNamespace('_' . $project->getInternalId()); + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } return $database; }; @@ -249,7 +305,7 @@ try { * Any worker can be configured with the following env vars: * - _APP_WORKERS_NUM The total number of worker processes * - _APP_WORKER_PER_CORE The number of worker processes per core (ignored if _APP_WORKERS_NUM is set) - * - _APP_QUEUE_NAME The name of the queue to read for database events + * - _APP_QUEUE_NAME The name of the queue to read for database events */ $platform->init(Service::TYPE_WORKER, [ 'workersNum' => System::getEnv('_APP_WORKERS_NUM', 1), diff --git a/composer.json b/composer.json index df004e6b44..8865a32d91 100644 --- a/composer.json +++ b/composer.json @@ -52,7 +52,7 @@ "utopia-php/config": "0.2.*", "utopia-php/database": "0.49.*", "utopia-php/domains": "0.5.*", - "utopia-php/dsn": "0.2.*", + "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", "utopia-php/fetch": "0.2.*", "utopia-php/image": "0.6.*", @@ -84,7 +84,7 @@ "ext-fileinfo": "*", "appwrite/sdk-generator": "0.38.*", "phpunit/phpunit": "9.5.20", - "swoole/ide-helper": "5.0.2", + "swoole/ide-helper": "5.1.2", "textalk/websocket": "1.5.7", "laravel/pint": "^1.14" }, diff --git a/composer.lock b/composer.lock index 9d4300f0a4..72bead0f5c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "28ecd4705def24ce61eee0f6676439d6", + "content-hash": "6ce62f5b54254e5023c5ace349a0ced7", "packages": [ { "name": "adhocore/jwt", @@ -1672,16 +1672,16 @@ }, { "name": "utopia-php/dsn", - "version": "0.2.0", + "version": "0.2.1", "source": { "type": "git", "url": "https://github.com/utopia-php/dsn.git", - "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7" + "reference": "42ee37a3d1785100b2f69091c9d4affadb6846eb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/dsn/zipball/c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", - "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", + "url": "https://api.github.com/repos/utopia-php/dsn/zipball/42ee37a3d1785100b2f69091c9d4affadb6846eb", + "reference": "42ee37a3d1785100b2f69091c9d4affadb6846eb", "shasum": "" }, "require": { @@ -1713,9 +1713,9 @@ ], "support": { "issues": "https://github.com/utopia-php/dsn/issues", - "source": "https://github.com/utopia-php/dsn/tree/0.2.0" + "source": "https://github.com/utopia-php/dsn/tree/0.2.1" }, - "time": "2023-11-02T12:01:43+00:00" + "time": "2024-05-07T02:01:25+00:00" }, { "name": "utopia-php/fetch", @@ -5142,16 +5142,16 @@ }, { "name": "swoole/ide-helper", - "version": "5.0.2", + "version": "5.1.2", "source": { "type": "git", "url": "https://github.com/swoole/ide-helper.git", - "reference": "16cfee44a6ec92254228c39bcab2fb8ae74cc2ea" + "reference": "33ec7af9111b76d06a70dd31191cc74793551112" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/swoole/ide-helper/zipball/16cfee44a6ec92254228c39bcab2fb8ae74cc2ea", - "reference": "16cfee44a6ec92254228c39bcab2fb8ae74cc2ea", + "url": "https://api.github.com/repos/swoole/ide-helper/zipball/33ec7af9111b76d06a70dd31191cc74793551112", + "reference": "33ec7af9111b76d06a70dd31191cc74793551112", "shasum": "" }, "type": "library", @@ -5168,9 +5168,9 @@ "description": "IDE help files for Swoole.", "support": { "issues": "https://github.com/swoole/ide-helper/issues", - "source": "https://github.com/swoole/ide-helper/tree/5.0.2" + "source": "https://github.com/swoole/ide-helper/tree/5.1.2" }, - "time": "2023-03-20T06:05:55+00:00" + "time": "2024-02-01T22:28:11+00:00" }, { "name": "symfony/polyfill-ctype", diff --git a/docker-compose.yml b/docker-compose.yml index 0f78f08d36..82c38ba269 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,6 @@ x-logging: &x-logging max-file: "5" max-size: "10m" -version: "3" - services: traefik: image: traefik:2.11 @@ -94,6 +92,7 @@ services: - app/http.php environment: - _APP_ENV + - _APP_EDITION - _APP_WORKER_PER_CORE - _APP_LOCALE - _APP_CONSOLE_WHITELIST_ROOT diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index 442cbe4bbc..c0c5ef2f43 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -111,14 +111,19 @@ class Database extends Event $client = new Client($this->queue, $this->connection); - return $client->enqueue([ - 'project' => $this->project, - 'user' => $this->user, - 'type' => $this->type, - 'collection' => $this->collection, - 'document' => $this->document, - 'database' => $this->database, - 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); + try { + $result = $client->enqueue([ + 'project' => $this->project, + 'user' => $this->user, + 'type' => $this->type, + 'collection' => $this->collection, + 'document' => $this->document, + 'database' => $this->database, + 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) + ]); + return $result; + } catch (\Throwable $th) { + return false; + } } } diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 92bc52561c..0bb5ca4650 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -31,7 +31,7 @@ class HTTPTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); $this->assertEquals('Appwrite', $response['headers']['server']); $this->assertEquals('GET, POST, PUT, PATCH, DELETE', $response['headers']['access-control-allow-methods']); - $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent', $response['headers']['access-control-allow-headers']); + $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Shared-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Appwrite-Session, X-Fallback-Cookies, X-Forwarded-For, X-Forwarded-User-Agent', $response['headers']['access-control-allow-headers']); $this->assertEquals('X-Appwrite-Session, X-Fallback-Cookies', $response['headers']['access-control-expose-headers']); $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']); $this->assertEquals('true', $response['headers']['access-control-allow-credentials']); diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 70bbedcb38..085d3ca01b 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -1296,8 +1296,6 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders())); - \var_dump($attributes['body']); - $this->assertEquals(0, $attributes['body']['total']); } @@ -1438,10 +1436,9 @@ class DatabasesCustomServerTest extends Scope } // Test indexLimit = 64 - // MariaDB, MySQL, and MongoDB create 5 indexes per new collection + // MariaDB, MySQL, and MongoDB create 6 indexes per new collection // Add up to the limit, then check if the next index throws IndexLimitException for ($i = 0; $i < 58; $i++) { - // $this->assertEquals(true, static::getDatabase()->createIndex('indexLimit', "index{$i}", Database::INDEX_KEY, ["test{$i}"], [16])); $index = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/indexes', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 64ee831181..39264386df 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -9,6 +9,7 @@ use Tests\E2E\General\UsageTest; use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideClient; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; @@ -3461,4 +3462,500 @@ class ProjectsConsoleClientTest extends Scope return $data; } + + public function testTenantIsolation(): void + { + // Create a team and a project + $team = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'teamId' => ID::unique(), + 'name' => 'Amazing Team', + ]); + + $teamId = $team['body']['$id']; + + // Project-level isolation + $project1 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-shared-tables' => false + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Application level isolation (shared tables) + $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-shared-tables' => true + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Project-level isolation + $project3 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-shared-tables' => false + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Application level isolation (shared tables) + $project4 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-shared-tables' => true + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Create and API key in each project + $key1 = $this->client->call(Client::METHOD_POST, '/projects/' . $project1['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'], + ]); + + $key2 = $this->client->call(Client::METHOD_POST, '/projects/' . $project2['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'], + ]); + + $key3 = $this->client->call(Client::METHOD_POST, '/projects/' . $project3['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'], + ]); + + $key4 = $this->client->call(Client::METHOD_POST, '/projects/' . $project4['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write', 'users.read', 'users.write'], + ]); + + // Create a database in each project + $database1 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + $database2 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + $database3 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + $database4 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + // Create a collection in each project + $collection1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => $database1['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + $collection2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + $collection3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => $database3['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + $collection4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => $database4['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + // Create an attribute in each project + $attribute1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => $database1['body']['$id'], + 'collectionId' => $collection1['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + $attribute2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => $collection2['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + $attribute3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => $database3['body']['$id'], + 'collectionId' => $collection3['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + $attribute4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => $database4['body']['$id'], + 'collectionId' => $collection4['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + // Wait for attributes + \sleep(2); + + // Create an index in each project + $index1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => $database1['body']['$id'], + 'collectionId' => $collection1['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute1['body']['key']], + ]); + + $index2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => $collection2['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute2['body']['key']], + ]); + + $index3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => $database3['body']['$id'], + 'collectionId' => $collection3['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute3['body']['key']], + ]); + + $index4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => $database4['body']['$id'], + 'collectionId' => $collection4['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute4['body']['key']], + ]); + + // Wait for indexes + \sleep(2); + + // Assert that each project has only 1 database, 1 collection, 1 attribute and 1 index + $databasesProject1 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject1['body']['total']); + $this->assertEquals(1, \count($databasesProject1['body']['databases'])); + + $databasesProject2 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject2['body']['total']); + $this->assertEquals(1, \count($databasesProject2['body']['databases'])); + + $databasesProject3 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject3['body']['total']); + $this->assertEquals(1, \count($databasesProject3['body']['databases'])); + + $databasesProject4 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject4['body']['total']); + $this->assertEquals(1, \count($databasesProject4['body']['databases'])); + + $collectionsProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject1['body']['total']); + $this->assertEquals(1, \count($collectionsProject1['body']['collections'])); + + $collectionsProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject2['body']['total']); + $this->assertEquals(1, \count($collectionsProject2['body']['collections'])); + + $collectionsProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject3['body']['total']); + $this->assertEquals(1, \count($collectionsProject3['body']['collections'])); + + $collectionsProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject4['body']['total']); + $this->assertEquals(1, \count($collectionsProject4['body']['collections'])); + + $attributesProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject1['body']['total']); + $this->assertEquals(1, \count($attributesProject1['body']['attributes'])); + + $attributesProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject2['body']['total']); + $this->assertEquals(1, \count($attributesProject2['body']['attributes'])); + + $attributesProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject3['body']['total']); + $this->assertEquals(1, \count($attributesProject3['body']['attributes'])); + + $attributesProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject4['body']['total']); + $this->assertEquals(1, \count($attributesProject4['body']['attributes'])); + + $indexesProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject1['body']['total']); + $this->assertEquals(1, \count($indexesProject1['body']['indexes'])); + + $indexesProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject2['body']['total']); + $this->assertEquals(1, \count($indexesProject2['body']['indexes'])); + + $indexesProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject3['body']['total']); + $this->assertEquals(1, \count($indexesProject3['body']['indexes'])); + + $indexesProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject4['body']['total']); + $this->assertEquals(1, \count($indexesProject4['body']['indexes'])); + + // Attempt to read cross-type resources + $collectionProject2WithProject1Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject2WithProject1Key['headers']['status-code']); + + $collectionProject1WithProject2Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject1WithProject2Key['headers']['status-code']); + + // Attempt to read cross-tenant resources + $collectionProject3WithProject1Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject3WithProject1Key['headers']['status-code']); + + $collectionProject1WithProject3Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject1WithProject3Key['headers']['status-code']); + + // Assert that shared project resources can have the same ID as they're unique on tenant + ID not just ID + $collection5 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => $collection4['body']['$id'], + 'name' => 'Amazing Collection', + ]); + + $this->assertEquals(201, $collection5['headers']['status-code']); + + // Assert that users across projects on shared tables can have the same email as they're unique on tenant + email not just email + $user1 = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'userId' => 'user', + 'email' => 'test@appwrite.io', + 'password' => 'password', + 'name' => 'Test User', + ]); + + $this->assertEquals(201, $user1['headers']['status-code']); + + $user2 = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'userId' => 'user', + 'email' => 'test@appwrite.io', + 'password' => 'password', + 'name' => 'Test User', + ]); + + $this->assertEquals(201, $user2['headers']['status-code']); + } } diff --git a/tests/e2e/Services/Projects/ProjectsCustomClientTest.php b/tests/e2e/Services/Projects/ProjectsCustomClientTest.php index 25140309af..b8d43fc544 100644 --- a/tests/e2e/Services/Projects/ProjectsCustomClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsCustomClientTest.php @@ -10,9 +10,4 @@ class ProjectsCustomClientTest extends Scope { use ProjectCustom; use SideClient; - - public function testMock() - { - $this->assertEquals(true, true); - } }