diff --git a/.env b/.env index 65fb54cb04..ea7f385c96 100644 --- a/.env +++ b/.env @@ -23,6 +23,11 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword +_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/CHANGES.md b/CHANGES.md index 340aec16d4..2c649ab5f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ## Bugs - Fix license detection for Flutter and Dart SDKs [#4435](https://github.com/appwrite/appwrite/pull/4435) +- Fix missing realtime event for create function deployment [#4574](https://github.com/appwrite/appwrite/pull/4574) # Version 1.0.3 ## Bugs diff --git a/Dockerfile b/Dockerfile index a7cae38502..3910bc61a1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ ENV PHP_REDIS_VERSION=5.3.7 \ PHP_IMAGICK_VERSION=3.7.0 \ PHP_YAML_VERSION=2.2.2 \ PHP_MAXMINDDB_VERSION=v1.11.0 \ + PHP_MEMCACHED_VERSION=v3.2.0 \ PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 " RUN \ @@ -52,6 +53,7 @@ RUN \ imagemagick \ imagemagick-dev \ libmaxminddb-dev \ + libmemcached-dev \ zstd-dev RUN docker-php-ext-install sockets @@ -125,6 +127,15 @@ RUN \ ./configure && \ make && make install +# Memcached Extension +FROM compile as memcached +RUN \ + git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \ + cd php-memcached && \ + phpize && \ + ./configure && \ + make && make install + # Zstd Compression FROM compile as zstd RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ @@ -134,7 +145,6 @@ RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ && ./configure --with-libzstd \ && make && make install - # Rust Extensions Compile Image FROM php:8.0.18-cli as rust_compile @@ -271,6 +281,7 @@ RUN \ && apk add --no-cache \ libstdc++ \ certbot \ + rsync \ brotli-dev \ yaml-dev \ imagemagick \ @@ -304,6 +315,7 @@ COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imag COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=scrypt /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ @@ -332,6 +344,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ diff --git a/app/config/collections.php b/app/config/collections.php index 6fc7ec1e37..b4fabf0e65 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -534,6 +534,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('database'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, @@ -992,7 +1003,7 @@ $collections = [ '$id' => ID::custom('secret'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 512, // var_dump of \bin2hex(\random_bytes(128)) => string(256) doubling for encryption + 'size' => 512, // Output of \bin2hex(\random_bytes(128)) => string(256) doubling for encryption 'signed' => true, 'required' => true, 'default' => null, diff --git a/app/config/variables.php b/app/config/variables.php index 9f3bc018e8..f529831192 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -306,6 +306,24 @@ return [ 'question' => '', 'filter' => 'password' ], + // [ + // 'name' => '_APP_CONNECTIONS_DB_PROJECT', + // 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ], + // [ + // 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + // 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ] ], ], [ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index fb91a4517d..9b83c6dfce 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -5,7 +5,6 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; use Appwrite\Auth\Validator\Phone; use Appwrite\Detector\Detector; -use Appwrite\Event\Audit; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Phone as EventPhone; @@ -39,7 +38,6 @@ use Utopia\Database\Validator\UID; use Utopia\Locale\Locale; use Utopia\Validator\ArrayList; use Utopia\Validator\Assoc; -use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -64,7 +62,7 @@ App::post('/v1/account') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_ACCOUNT) ->label('abuse-limit', 10) - ->param('userId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password. Must be at least 8 chars.') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) @@ -141,7 +139,7 @@ App::post('/v1/account') App::post('/v1/account/sessions/email') ->alias('/v1/account/sessions') - ->desc('Create Account Session with Email') + ->desc('Create Email Session') ->groups(['api', 'account', 'auth']) ->label('event', 'users.[userId].sessions.[sessionId].create') ->label('scope', 'public') @@ -255,7 +253,7 @@ App::post('/v1/account/sessions/email') }); App::get('/v1/account/sessions/oauth2/:provider') - ->desc('Create Account Session with OAuth2') + ->desc('Create OAuth2 Session') ->groups(['api', 'account']) ->label('error', __DIR__ . '/../../views/general/error.phtml') ->label('scope', 'public') @@ -619,7 +617,7 @@ App::post('/v1/account/sessions/magic-url') ->label('sdk.response.model', Response::MODEL_TOKEN) ->label('abuse-limit', 10) ->label('abuse-key', 'url:{url},email:{param-email}') - ->param('userId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients']) ->inject('request') @@ -872,7 +870,7 @@ App::post('/v1/account/sessions/phone') ->label('sdk.response.model', Response::MODEL_TOKEN) ->label('abuse-limit', 10) ->label('abuse-key', 'url:{url},email:{param-email}') - ->param('userId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('phone', '', new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.') ->inject('request') ->inject('response') @@ -1223,7 +1221,7 @@ App::post('/v1/account/sessions/anonymous') }); App::post('/v1/account/jwt') - ->desc('Create Account JWT') + ->desc('Create JWT') ->groups(['api', 'account', 'auth']) ->label('scope', 'account') ->label('auth.type', 'jwt') @@ -1310,7 +1308,7 @@ App::get('/v1/account/prefs') }); App::get('/v1/account/sessions') - ->desc('List Account Sessions') + ->desc('List Sessions') ->groups(['api', 'account']) ->label('scope', 'account') ->label('usage.metric', 'users.{scope}.requests.read') @@ -1345,7 +1343,7 @@ App::get('/v1/account/sessions') }); App::get('/v1/account/logs') - ->desc('List Account Logs') + ->desc('List Logs') ->groups(['api', 'account']) ->label('scope', 'account') ->label('usage.metric', 'users.{scope}.requests.read') @@ -1406,7 +1404,7 @@ App::get('/v1/account/logs') }); App::get('/v1/account/sessions/:sessionId') - ->desc('Get Session By ID') + ->desc('Get Session') ->groups(['api', 'account']) ->label('scope', 'account') ->label('usage.metric', 'users.{scope}.requests.read') @@ -1446,7 +1444,7 @@ App::get('/v1/account/sessions/:sessionId') }); App::patch('/v1/account/name') - ->desc('Update Account Name') + ->desc('Update Name') ->groups(['api', 'account']) ->label('event', 'users.[userId].update.name') ->label('scope', 'account') @@ -1477,7 +1475,7 @@ App::patch('/v1/account/name') }); App::patch('/v1/account/password') - ->desc('Update Account Password') + ->desc('Update Password') ->groups(['api', 'account']) ->label('event', 'users.[userId].update.password') ->label('scope', 'account') @@ -1517,7 +1515,7 @@ App::patch('/v1/account/password') }); App::patch('/v1/account/email') - ->desc('Update Account Email') + ->desc('Update Email') ->groups(['api', 'account']) ->label('event', 'users.[userId].update.email') ->label('scope', 'account') @@ -1569,7 +1567,7 @@ App::patch('/v1/account/email') }); App::patch('/v1/account/phone') - ->desc('Update Account Phone') + ->desc('Update Phone') ->groups(['api', 'account']) ->label('event', 'users.[userId].update.phone') ->label('scope', 'account') @@ -1617,7 +1615,7 @@ App::patch('/v1/account/phone') }); App::patch('/v1/account/prefs') - ->desc('Update Account Preferences') + ->desc('Update Preferences') ->groups(['api', 'account']) ->label('event', 'users.[userId].update.prefs') ->label('scope', 'account') @@ -1646,7 +1644,7 @@ App::patch('/v1/account/prefs') }); App::patch('/v1/account/status') - ->desc('Update Account Status') + ->desc('Update Status') ->groups(['api', 'account']) ->label('event', 'users.[userId].update.status') ->label('scope', 'account') @@ -1681,7 +1679,7 @@ App::patch('/v1/account/status') }); App::delete('/v1/account/sessions/:sessionId') - ->desc('Delete Account Session') + ->desc('Delete Session') ->groups(['api', 'account']) ->label('scope', 'account') ->label('event', 'users.[userId].sessions.[sessionId].delete') @@ -1752,7 +1750,7 @@ App::delete('/v1/account/sessions/:sessionId') }); App::patch('/v1/account/sessions/:sessionId') - ->desc('Update Session (Refresh Tokens)') + ->desc('Update OAuth Session (Refresh Tokens)') ->groups(['api', 'account']) ->label('scope', 'account') ->label('event', 'users.[userId].sessions.[sessionId].update') @@ -1834,7 +1832,7 @@ App::patch('/v1/account/sessions/:sessionId') }); App::delete('/v1/account/sessions') - ->desc('Delete All Account Sessions') + ->desc('Delete Sessions') ->groups(['api', 'account']) ->label('scope', 'account') ->label('event', 'users.[userId].sessions.[sessionId].delete') @@ -1886,8 +1884,6 @@ App::delete('/v1/account/sessions') $dbForProject->deleteCachedDocument('users', $user->getId()); - $numOfSessions = count($sessions); - $events ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 19264454e0..2b791e6ea9 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -20,7 +20,6 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\DateTime; use Utopia\Database\Query; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Key; use Utopia\Database\Validator\Permissions; @@ -163,7 +162,7 @@ App::post('/v1/databases') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_DATABASE) // Model for database needs to be created - ->param('databaseId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('databaseId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') ->inject('response') ->inject('dbForProject') @@ -489,7 +488,7 @@ App::post('/v1/databases/:databaseId/collections') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_COLLECTION) ->param('databaseId', '', new UID(), 'Database ID.') - ->param('collectionId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('collectionId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default no user is granted with any permissions. [Learn more about permissions](/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](/docs/permissions).', true) @@ -1574,7 +1573,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') Query::equal('databaseInternalId', [$db->getInternalId()]) ], 61); - $limit = 64 - MariaDB::getCountOfDefaultIndexes(); + $limit = $dbForProject->getLimitForIndexes(); if ($count >= $limit) { throw new Exception(Exception::INDEX_LIMIT_EXCEEDED, 'Index limit exceeded'); @@ -1855,7 +1854,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_DOCUMENT) ->param('databaseId', '', new UID(), 'Database ID.') - ->param('documentId', '', new CustomId(), 'Document ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('documentId', '', new CustomId(), 'Document ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') ->param('data', [], new JSON(), 'Document data as JSON object.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default the current user is granted with all permissions. [Learn more about permissions](/docs/permissions).', true) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d3a26b414e..188c5e453c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -25,7 +25,6 @@ use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Database\Validator\Queries\Deployments; use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Database\Validator\Queries\Functions; -use Appwrite\Utopia\Database\Validator\Queries\Variables; use Utopia\App; use Utopia\Database\Database; use Utopia\Database\Document; @@ -33,7 +32,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Validator\ArrayList; -use Utopia\Validator\Assoc; use Utopia\Validator\Text; use Utopia\Validator\Range; use Utopia\Validator\WhiteList; @@ -61,7 +59,7 @@ App::post('/v1/functions') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_FUNCTION) - ->param('functionId', '', new CustomId(), 'Function ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('functionId', '', new CustomId(), 'Function ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', '', new Text(128), 'Function name. Max length: 128 chars.') ->param('execute', [], new Roles(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of strings with execution roles. By default no user is granted with any execute permissions. [learn more about permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 64 characters long.') ->param('runtime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Execution runtime.') diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4384fcbcd5..f65e65ba23 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -5,7 +5,9 @@ use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\Utopia\Response; use Utopia\App; +use Utopia\Config\Config; use Utopia\Database\Document; +use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -26,6 +28,7 @@ App::get('/v1/health') ->action(function (Response $response) { $output = [ + 'name' => 'http', 'status' => 'pass', 'ping' => 0 ]; @@ -42,7 +45,6 @@ App::get('/v1/health/version') ->label('sdk.response.model', Response::MODEL_HEALTH_VERSION) ->inject('response') ->action(function (Response $response) { - $response->dynamic(new Document([ 'version' => APP_VERSION_STABLE ]), Response::MODEL_HEALTH_VERSION); }); @@ -58,30 +60,50 @@ App::get('/v1/health/db') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - try { - $db = $utopia->getResource('db'); /* @var $db PDO */ - - // Run a small test to check the connection - $statement = $db->prepare("SELECT 1;"); - - $statement->closeCursor(); - - $statement->execute(); - } catch (Exception $_e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Database is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/cache') @@ -96,23 +118,163 @@ App::get('/v1/health/cache') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - $redis = $utopia->getResource('cache'); - - if (!$redis->ping(true)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Cache is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Cache' => Config::getParam('pools-cache'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/queue') + ->desc('Get Queue') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getQueue') + ->label('sdk.description', '/docs/references/health/get-queue.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'Queue' => Config::getParam('pools-queue'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/pubsub') + ->desc('Get PubSub') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getPubSub') + ->label('sdk.description', '/docs/references/health/get-pubsub.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'PubSub' => Config::getParam('pools-pubsub'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/time') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 360f0be814..3935a71a9d 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -29,6 +29,8 @@ use Utopia\Domains\Domain; use Utopia\Registry\Registry; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\Queries\Projects; +use Utopia\Cache\Cache; +use Utopia\Pools\Group; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; @@ -55,7 +57,7 @@ App::post('/v1/projects') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_PROJECT) - ->param('projectId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('projectId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) @@ -69,8 +71,10 @@ App::post('/v1/projects') ->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true) ->inject('response') ->inject('dbForConsole') - ->inject('dbForProject') - ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Database $dbForProject) { + ->inject('cache') + ->inject('pools') + ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { + $team = $dbForConsole->getDocument('teams', $teamId); @@ -85,6 +89,8 @@ App::post('/v1/projects') } $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId; + $databases = Config::getParam('pools-database', []); + $database = $databases[array_rand($databases)]; if ($projectId === 'console') { throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); @@ -120,10 +126,10 @@ App::post('/v1/projects') 'domains' => null, 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), + 'database' => $database, ])); - /** @var array $collections */ - $collections = Config::getParam('collections', []); + $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); $dbForProject->setNamespace("_{$project->getInternalId()}"); $dbForProject->create(); @@ -133,6 +139,9 @@ App::post('/v1/projects') $adapter = new TimeLimit('', 0, 1, $dbForProject); $adapter->setup(); + /** @var array $collections */ + $collections = Config::getParam('collections', []); + foreach ($collections as $key => $collection) { if (($collection['$collection'] ?? '') !== Database::METADATA) { continue; diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f236285749..530f174cd0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -58,7 +58,7 @@ App::post('/v1/storage/buckets') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_BUCKET) - ->param('bucketId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('bucketId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', '', new Text(128), 'Bucket name') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default no user is granted with any permissions. [Learn more about permissions](/docs/permissions).', true) ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](/docs/permissions).', true) @@ -347,7 +347,7 @@ App::post('/v1/storage/buckets/:bucketId/files') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_FILE) ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') - ->param('fileId', '', new CustomId(), 'File ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('fileId', '', new CustomId(), 'File ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('file', [], new File(), 'Binary file.', false) ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permission strings. By default the current user is granted with all permissions. [Learn more about permissions](/docs/permissions).', true) ->inject('request') diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index a42b9d317e..b87017dbd2 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -20,7 +20,6 @@ use Appwrite\Utopia\Response; use MaxMind\Db\Reader; use Utopia\App; use Utopia\Audit\Audit; -use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -36,9 +35,7 @@ use Utopia\Database\Validator\Key; use Utopia\Database\Validator\UID; use Utopia\Locale\Locale; use Utopia\Validator\Text; -use Utopia\Validator\Range; use Utopia\Validator\ArrayList; -use Utopia\Validator\WhiteList; App::post('/v1/teams') ->desc('Create Team') @@ -54,7 +51,7 @@ App::post('/v1/teams') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_TEAM) - ->param('teamId', '', new CustomId(), 'Team ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('teamId', '', new CustomId(), 'Team ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Team name. Max length: 128 chars.') ->param('roles', ['owner'], new ArrayList(new Key(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the roles in the team for the user who created it. The default role is **owner**. A role can be any string. Learn more about [roles and permissions](/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 32 characters long.', true) ->inject('response') diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index c95105b775..8c8e4143ab 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -98,7 +98,7 @@ App::post('/v1/users') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', null, new Email(), 'User email.', true) ->param('phone', null, new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.', true) ->param('password', null, new Password(), 'Plain text user password. Must be at least 8 chars.', true) @@ -129,7 +129,7 @@ App::post('/v1/users/bcrypt') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password hashed using Bcrypt.') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) @@ -159,7 +159,7 @@ App::post('/v1/users/md5') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password hashed using MD5.') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) @@ -189,7 +189,7 @@ App::post('/v1/users/argon2') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password hashed using Argon2.') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) @@ -219,7 +219,7 @@ App::post('/v1/users/sha') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password hashed using SHA.') ->param('passwordVersion', '', new WhiteList(['sha1', 'sha224', 'sha256', 'sha384', 'sha512/224', 'sha512/256', 'sha512', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512']), "Optional SHA version used to hash password. Allowed values are: 'sha1', 'sha224', 'sha256', 'sha384', 'sha512/224', 'sha512/256', 'sha512', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512'", true) @@ -256,7 +256,7 @@ App::post('/v1/users/phpass') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()`to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password hashed using PHPass.') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) @@ -286,7 +286,7 @@ App::post('/v1/users/scrypt') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password hashed using Scrypt.') ->param('passwordSalt', '', new Text(128), 'Optional salt used to hash password.') @@ -329,7 +329,7 @@ App::post('/v1/users/scrypt-modified') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USER) - ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('userId', '', new CustomId(), 'User ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('email', '', new Email(), 'User email.') ->param('password', '', new Password(), 'User password hashed using Scrypt Modified.') ->param('passwordSalt', '', new Text(128), 'Salt used to hash password.') diff --git a/app/http.php b/app/http.php index f27bf82353..b497c466f3 100644 --- a/app/http.php +++ b/app/http.php @@ -22,6 +22,7 @@ use Utopia\Swoole\Files; use Appwrite\Utopia\Request; use Utopia\Logger\Log; use Utopia\Logger\Log\User; +use Utopia\Pools\Group; $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); @@ -60,6 +61,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $app = new App('UTC'); go(function () use ($register, $app) { + $pools = $register->get('pools'); /** @var Group $pools */ + App::setResource('pools', fn() => $pools); + // wait for database to be ready $attempts = 0; $max = 10; @@ -68,8 +72,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { do { try { $attempts++; - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); + $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ break; // leave the do-while if successful } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); @@ -80,19 +83,14 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { } } while ($attempts < $max); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); - - /** @var Utopia\Database\Database $dbForConsole */ - $dbForConsole = $app->getResource('dbForConsole'); - Console::success('[Setup] - Server database init started...'); /** @var array $collections */ $collections = Config::getParam('collections', []); try { - $redis->flushAll(); + $cache = $app->getResource('cache'); /** @var Utopia\Cache\Cache $cache */ + $cache->flush(); Console::success('[Setup] - Creating database: appwrite...'); $dbForConsole->create(); } catch (\Exception $e) { @@ -116,10 +114,11 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { if (!$dbForConsole->getCollection($key)->isEmpty()) { continue; } + /** * Skip to prevent 0.16 migration issues. */ - if (in_array($key, ['cache', 'variables']) && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { + if (in_array($key, ['cache', 'variables']) && $dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) { continue; } @@ -155,7 +154,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection($key, $attributes, $indexes); } - if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { + if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) { Console::success('[Setup] - Creating default bucket...'); $dbForConsole->createDocument('buckets', new Document([ '$id' => ID::custom('default'), @@ -215,6 +214,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } + $pools->reclaim(); + Console::success('[Setup] - Server database init completed...'); }); @@ -246,11 +247,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $app = new App('UTC'); - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); + $pools = $register->get('pools'); + App::setResource('pools', fn() => $pools); try { Authorization::cleanRoles(); @@ -334,13 +332,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - /** @var PDOPool $dbPool */ - $dbPool = $register->get('dbPool'); - $dbPool->put($db); - - /** @var RedisPool $redisPool */ - $redisPool = $register->get('redisPool'); - $redisPool->put($redis); + $pools->reclaim(); } }); diff --git a/app/init.php b/app/init.php index 054721d439..09d5ce0589 100644 --- a/app/init.php +++ b/app/init.php @@ -18,9 +18,6 @@ ini_set('display_startup_errors', 1); ini_set('default_socket_timeout', -1); error_reporting(E_ALL); -use Appwrite\Extend\PDO; -use Ahc\Jwt\JWT; -use Ahc\Jwt\JWTException; use Appwrite\Extend\Exception; use Appwrite\Auth\Auth; use Appwrite\SMS\Adapter\Mock; @@ -32,39 +29,31 @@ use Appwrite\SMS\Adapter\Vonage; use Appwrite\DSN\DSN; use Appwrite\Event\Audit; use Appwrite\Event\Database as EventDatabase; -use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Phone; +use Appwrite\Event\Delete; use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\URL\URL as AppwriteURL; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; +use Utopia\Validator\Range; +use Utopia\Validator\WhiteList; use Utopia\Database\ID; +use Utopia\Database\Document; +use Utopia\Database\Database; +use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\DatetimeValidator; +use Utopia\Database\Validator\Structure; use Utopia\Logger\Logger; use Utopia\Config\Config; use Utopia\Locale\Locale; use Utopia\Registry\Registry; -use MaxMind\Db\Reader; -use PHPMailer\PHPMailer\PHPMailer; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Document; -use Utopia\Database\Database; -use Utopia\Database\Validator\Structure; -use Utopia\Database\Validator\Authorization; -use Utopia\Validator\Range; -use Utopia\Validator\WhiteList; -use Swoole\Database\PDOConfig; -use Swoole\Database\PDOPool; -use Swoole\Database\RedisConfig; -use Swoole\Database\RedisPool; -use Utopia\Database\Query; -use Utopia\Database\Validator\DatetimeValidator; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Backblaze; @@ -73,6 +62,18 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; +use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\MySQL; +use Utopia\Pools\Group; +use Utopia\Pools\Pool; +use Ahc\Jwt\JWT; +use Ahc\Jwt\JWTException; +use MaxMind\Db\Reader; +use PHPMailer\PHPMailer\PHPMailer; +use Swoole\Database\PDOProxy; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -495,56 +496,173 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); -$register->set('dbPool', function () { - // Register DB connection - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbPort = App::getEnv('_APP_DB_PORT', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); +$register->set('pools', function () { + $group = new Group(); - $pool = new PDOPool( - (new PDOConfig()) - ->withHost($dbHost) - ->withPort($dbPort) - ->withDbName($dbScheme) - ->withCharset('utf8mb4') - ->withUsername($dbUser) - ->withPassword($dbPass) - ->withOptions([ - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - ]), - 64 - ); + $fallbackForDB = AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => App::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => App::getEnv('_APP_DB_PORT', '3306'), + 'user' => App::getEnv('_APP_DB_USER', ''), + 'pass' => App::getEnv('_APP_DB_PASS', ''), + ]); + $fallbackForRedis = AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); - return $pool; -}); -$register->set('redisPool', function () { - $redisHost = App::getEnv('_APP_REDIS_HOST', ''); - $redisPort = App::getEnv('_APP_REDIS_PORT', ''); - $redisUser = App::getEnv('_APP_REDIS_USER', ''); - $redisPass = App::getEnv('_APP_REDIS_PASS', ''); - $redisAuth = ''; + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; - if ($redisUser && $redisPass) { - $redisAuth = $redisUser . ':' . $redisPass; + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $dsns = $connection['dsns'] ?? ''; + $multipe = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getDatabase(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused accross connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + + switch ($dsnScheme) { + case 'mysql': + case 'mariadb': + $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }; + break; + case 'redis': + $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); + break; + } + + $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) { + // Get Adapter + $adapter = null; + + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + $adapter->setDefaultDatabase($dsn->getDatabase()); + + break; + case 'queue': + $adapter = $resource(); + break; + case 'pubsub': + $adapter = $resource(); + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + break; + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); } - $pool = new RedisPool( - (new RedisConfig()) - ->withHost($redisHost) - ->withPort($redisPort) - ->withAuth($redisAuth) - ->withDbIndex(0), - 64 - ); - - return $pool; + return $group; }); $register->set('influxdb', function () { // Register DB connection @@ -561,7 +679,7 @@ $register->set('influxdb', function () { return $client; }); $register->set('statsd', function () { - // Register DB connection + // Register DB connection $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); @@ -601,33 +719,6 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__ . '/db/DBIP/dbip-country-lite-2022-06.mmdb'); }); -$register->set('db', function () { - // This is usually for our workers or CLI commands scope - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbPort = App::getEnv('_APP_DB_PORT', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - - $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - )); - - return $pdo; -}); -$register->set('cache', function () { - // This is usually for our workers or CLI commands scope - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); /* * Localization @@ -925,26 +1016,51 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function ($db, $cache, Document $project) { - $cache = new Cache(new RedisCache($cache)); +App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_{$project->getInternalId()}"); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('_' . $project->getInternalId()); return $database; -}, ['db', 'cache', 'project']); +}, ['pools', 'dbForConsole', 'cache', 'project']); -App::setResource('dbForConsole', function ($db, $cache) { - $cache = new Cache(new RedisCache($cache)); +App::setResource('dbForConsole', function (Group $pools, Cache $cache) { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); + $database = new Database($dbAdapter, $cache); + + $database->setNamespace('console'); return $database; -}, ['db', 'cache']); +}, ['pools', 'cache']); +App::setResource('cache', function (Group $pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); App::setResource('deviceLocal', function () { return new Local(); diff --git a/app/preload.php b/app/preload.php index 8789936417..4935db3da4 100644 --- a/app/preload.php +++ b/app/preload.php @@ -35,7 +35,7 @@ foreach ( realpath(__DIR__ . '/../vendor/symfony'), realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload - realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcache autoload + realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload ] as $key => $value ) { if ($value !== false) { diff --git a/app/realtime.php b/app/realtime.php index be87c3d6e6..f0a2b6e4a2 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -16,16 +16,15 @@ use Utopia\CLI\Console; use Utopia\Database\ID; use Utopia\Database\Role; use Utopia\Logger\Log; -use Utopia\Database\Database; use Utopia\Database\DateTime; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Appwrite\Utopia\Request; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\WebSocket\Server; use Utopia\WebSocket\Adapter; @@ -33,6 +32,67 @@ require_once __DIR__ . '/init.php'; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); +function getConsoleDB(): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +function getProjectDB(Document $project): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +} + $realtime = new Realtime(); /** @@ -95,45 +155,6 @@ $logError = function (Throwable $error, string $action) use ($register) { $server->error($logError); -function getDatabase(Registry &$register, string $namespace) -{ - $attempts = 0; - - do { - try { - $attempts++; - - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'realtime')) { - throw new Exception('Collection not ready'); - } - - break; // leave loop if successful - } catch (\Throwable $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return [ - $database, - function () use ($register, $db, $redis) { - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - } - ]; -} - $server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) { sleep(5); // wait for the initial database schema to be ready Console::success('Server started successfully'); @@ -143,7 +164,8 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume */ go(function () use ($register, $containerId, &$statsDocument) { $attempts = 0; - [$database, $returnDatabase] = getDatabase($register, '_console'); + $database = getConsoleDB(); + do { try { $attempts++; @@ -163,7 +185,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume sleep(DATABASE_RECONNECT_SLEEP); } } while (true); - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); }); /** @@ -179,7 +201,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } try { - [$database, $returnDatabase] = getDatabase($register, '_console'); + $database = getConsoleDB(); $statsDocument ->setAttribute('timestamp', DateTime::now()) @@ -189,7 +211,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } catch (\Throwable $th) { call_user_func($logError, $th, "updateWorkerDocument"); } finally { - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } }); }); @@ -205,7 +227,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, * Sending current connections to project channels on the console project every 5 seconds. */ if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) { - [$database, $returnDatabase] = getDatabase($register, '_console'); + $database = getConsoleDB(); $payload = []; @@ -250,7 +272,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, ])); } - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } /** * Sending test message for SDK E2E tests every 5 seconds. @@ -285,8 +307,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $start = time(); - /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); + $redis = $register->get('pools')->get('pubsub')->pop()->getResource(); /** @var Redis $redis */ $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); if ($redis->ping(true)) { @@ -305,9 +326,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) { $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId])); - [$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, '_console'); + $consoleDatabase = getConsoleDB(); $project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId)); - [$database, $returnDatabase] = getDatabase($register, "_{$project->getInternalId()}"); + $database = getProjectDB($project); $user = $database->getDocument('users', $userId); @@ -315,8 +336,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']); - call_user_func($returnDatabase); - call_user_func($returnConsoleDatabase); + $register->get('pools')->reclaim(); } } @@ -344,7 +364,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, call_user_func($logError, $th, "pubSubConnection"); Console::error('Pub/sub error: ' . $th->getMessage()); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); $attempts++; sleep(DATABASE_RECONNECT_SLEEP); continue; @@ -359,33 +379,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $request = new Request($request); $response = new Response(new SwooleResponse()); - /** @var PDO $db */ - $db = $register->get('dbPool')->get(); - /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); - Console::info("Connection open (user: {$connection})"); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); - App::setResource('request', fn () => $request); - App::setResource('response', fn () => $response); + App::setResource('pools', fn() => $register->get('pools')); + App::setResource('request', fn() => $request); + App::setResource('response', fn() => $response); try { - /** @var \Utopia\Database\Document $user */ - $user = $app->getResource('user'); - /** @var \Utopia\Database\Document $project */ $project = $app->getResource('project'); - /** @var \Utopia\Database\Document $console */ - $console = $app->getResource('console'); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_{$project->getInternalId()}"); - /* * Project Check */ @@ -393,12 +396,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception('Missing or unknown project ID', 1008); } + $dbForProject = getProjectDB($project); + $console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */ + $user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */ + /* * Abuse Check * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $database); + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject); $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); @@ -469,34 +476,20 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, Console::error('[Error] Code: ' . $response['data']['code']); Console::error('[Error] Message: ' . $response['data']['message']); } - - if ($th instanceof PDOException) { - $db = null; - } } finally { - /** - * Put used PDO and Redis Connections back into their pools. - */ - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); } }); $server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) { try { $response = new Response(new SwooleResponse()); - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_console"); $projectId = $realtime->connections[$connection]['projectId']; + $database = getConsoleDB(); if ($projectId !== 'console') { $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId)); - $database->setNamespace("_{$project->getInternalId()}"); + $database = getProjectDB($project); } /* @@ -580,8 +573,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $server->close($connection, $th->getCode()); } } finally { - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); } }); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 0442d8456b..1d7fdcd046 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -88,15 +88,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -184,13 +184,15 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -209,15 +211,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -263,15 +265,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -312,15 +314,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -340,15 +342,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -372,15 +374,15 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -399,15 +401,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -535,15 +537,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -571,14 +573,14 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -603,14 +605,14 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG diff --git a/app/workers/audits.php b/app/workers/audits.php index b86649543b..90ac020536 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -1,6 +1,5 @@ getAttribute('name', ''); $userEmail = $user->getAttribute('email', ''); - $dbForProject = $this->getProjectDB($project->getId()); + $dbForProject = $this->getProjectDB($project); $audit = new Audit($dbForProject); $audit->log( userId: $user->getId(), diff --git a/app/workers/builds.php b/app/workers/builds.php index bf780c6464..235ad0da72 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -7,7 +7,6 @@ use Appwrite\Utopia\Response\Model\Deployment; use Cron\CronExpression; use Executor\Executor; use Appwrite\Usage\Stats; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\App; use Utopia\CLI\Console; @@ -15,7 +14,6 @@ use Utopia\Database\ID; use Utopia\Storage\Storage; use Utopia\Database\Document; use Utopia\Config\Config; -use Utopia\Database\Query; require_once __DIR__ . '/../init.php'; @@ -59,7 +57,7 @@ class BuildsV1 extends Worker protected function buildDeployment(Document $project, Document $function, Document $deployment) { - $dbForProject = $this->getProjectDB($project->getId()); + $dbForProject = $this->getProjectDB($project); $function = $dbForProject->getDocument('functions', $function->getId()); if ($function->isEmpty()) { diff --git a/app/workers/certificates.php b/app/workers/certificates.php index e7885bc669..b4f0701c46 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -1,6 +1,5 @@ createAttribute($database, $collection, $document, $project->getId()); + $this->createAttribute($database, $collection, $document, $project); break; case DATABASE_TYPE_DELETE_ATTRIBUTE: - $this->deleteAttribute($database, $collection, $document, $project->getId()); + $this->deleteAttribute($database, $collection, $document, $project); break; case DATABASE_TYPE_CREATE_INDEX: - $this->createIndex($database, $collection, $document, $project->getId()); + $this->createIndex($database, $collection, $document, $project); break; case DATABASE_TYPE_DELETE_INDEX: - $this->deleteIndex($database, $collection, $document, $project->getId()); + $this->deleteIndex($database, $collection, $document, $project); break; default: @@ -61,12 +61,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $attribute - * @param string $projectId + * @param Document $project */ - protected function createAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void + protected function createAttribute(Document $database, Document $collection, Document $attribute, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ 'databaseId' => $database->getId(), @@ -128,12 +129,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $attribute - * @param string $projectId + * @param Document $project */ - protected function deleteAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void + protected function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ 'databaseId' => $database->getId(), @@ -225,7 +227,7 @@ class DatabaseV1 extends Worker } if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $projectId); + $this->deleteIndex($database, $collection, $index, $project); } else { $dbForProject->updateDocument('indexes', $index->getId(), $index); } @@ -241,12 +243,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $index - * @param string $projectId + * @param Document $project */ - protected function createIndex(Document $database, Document $collection, Document $index, string $projectId): void + protected function createIndex(Document $database, Document $collection, Document $index, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ 'databaseId' => $database->getId(), @@ -298,12 +301,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $index - * @param string $projectId + * @param Document $project */ - protected function deleteIndex(Document $database, Document $collection, Document $index, string $projectId): void + protected function deleteIndex(Document $database, Document $collection, Document $index, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ 'databaseId' => $database->getId(), diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b015043b1d..9ef593dbb9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -40,28 +40,28 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { case DELETE_TYPE_DATABASES: - $this->deleteDatabase($document, $project->getId()); + $this->deleteDatabase($document, $project); break; case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($document, $project->getId()); + $this->deleteCollection($document, $project); break; case DELETE_TYPE_PROJECTS: $this->deleteProject($document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($document, $project->getId()); + $this->deleteFunction($document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($document, $project->getId()); + $this->deleteDeployment($document, $project); break; case DELETE_TYPE_USERS: - $this->deleteUser($document, $project->getId()); + $this->deleteUser($document, $project); break; case DELETE_TYPE_TEAMS: - $this->deleteMemberships($document, $project->getId()); + $this->deleteMemberships($document, $project); break; case DELETE_TYPE_BUCKETS: - $this->deleteBucket($document, $project->getId()); + $this->deleteBucket($document, $project); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -82,7 +82,7 @@ class DeletesV1 extends Worker $document = new Document($this->args['document'] ?? []); if (!$document->isEmpty()) { - $this->deleteAuditLogsByResource('document/' . $document->getId(), $project->getId()); + $this->deleteAuditLogsByResource('document/' . $document->getId(), $project); } break; @@ -109,7 +109,7 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($project->getId()); + $this->deleteCacheByResource($this->args['resource']); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate(); @@ -125,12 +125,12 @@ class DeletesV1 extends Worker } /** - * @param string $projectId + * @param string $resource */ - protected function deleteCacheByResource(string $projectId): void + protected function deleteCacheByResource(string $resource): void { $this->deleteCacheFiles([ - Query::equal('resource', [$this->args['resource']]), + Query::equal('resource', [$resource]), ]); } @@ -143,9 +143,10 @@ class DeletesV1 extends Worker protected function deleteCacheFiles($query): void { - $this->deleteForProjectIds(function (string $projectId) use ($query) { + $this->deleteForProjectIds(function (Document $project) use ($query) { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); @@ -169,34 +170,35 @@ class DeletesV1 extends Worker /** * @param Document $document database document - * @param string $projectId + * @param Document $projectId */ - protected function deleteDatabase(Document $document, string $projectId): void + protected function deleteDatabase(Document $document, Document $project): void { $databaseId = $document->getId(); + $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($projectId) { - $this->deleteCollection($document, $projectId); + $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { + $this->deleteCollection($document, $project); }); $dbForProject->deleteCollection('database_' . $document->getInternalId()); - $this->deleteAuditLogsByResource('database/' . $databaseId, $projectId); + $this->deleteAuditLogsByResource('database/' . $databaseId, $project); } /** * @param Document $document teams document - * @param string $projectId + * @param Document $project */ - protected function deleteCollection(Document $document, string $projectId): void + protected function deleteCollection(Document $document, Document $project): void { $collectionId = $document->getId(); $databaseId = $document->getAttribute('databaseId'); $databaseInternalId = $document->getAttribute('databaseInternalId'); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); @@ -210,7 +212,7 @@ class DeletesV1 extends Worker Query::equal('collectionId', [$collectionId]) ], $dbForProject); - $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $projectId); + $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); } /** @@ -219,8 +221,8 @@ class DeletesV1 extends Worker */ protected function deleteUsageStats(string $datetime1d, string $datetime30m) { - $this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime30m) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime1d, $datetime30m) { + $dbForProject = $this->getProjectDB($project); // Delete Usage stats $this->deleteByGroup('stats', [ Query::lessThan('time', $datetime1d), @@ -236,16 +238,16 @@ class DeletesV1 extends Worker /** * @param Document $document teams document - * @param string $projectId + * @param Document $project */ - protected function deleteMemberships(Document $document, string $projectId): void + protected function deleteMemberships(Document $document, Document $project): void { $teamId = $document->getAttribute('teamId', ''); // Delete Memberships $this->deleteByGroup('memberships', [ Query::equal('teamId', [$teamId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); } /** @@ -256,7 +258,7 @@ class DeletesV1 extends Worker $projectId = $document->getId(); // Delete all DBs - $this->getProjectDB($projectId)->delete($projectId); + $this->getProjectDB($document)->delete($projectId); // Delete all storage directories $uploads = new Local(APP_STORAGE_UPLOADS . '/app-' . $document->getId()); @@ -268,30 +270,30 @@ class DeletesV1 extends Worker /** * @param Document $document user document - * @param string $projectId + * @param Document $project */ - protected function deleteUser(Document $document, string $projectId): void + protected function deleteUser(Document $document, Document $project): void { $userId = $document->getId(); // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); - $this->getProjectDB($projectId)->deleteCachedDocument('users', $userId); + $this->getProjectDB($project)->deleteCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId), function (Document $document) use ($projectId) { + ], $this->getProjectDB($project), function (Document $document) use ($project) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); - $team = $this->getProjectDB($projectId)->getDocument('teams', $teamId); + $team = $this->getProjectDB($project)->getDocument('teams', $teamId); if (!$team->isEmpty()) { $team = $this - ->getProjectDB($projectId) + ->getProjectDB($project) ->updateDocument( 'teams', $teamId, @@ -305,7 +307,7 @@ class DeletesV1 extends Worker // Delete tokens $this->deleteByGroup('tokens', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); } /** @@ -313,8 +315,8 @@ class DeletesV1 extends Worker */ protected function deleteExecutionLogs(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Executions $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime) @@ -327,8 +329,8 @@ class DeletesV1 extends Worker */ protected function deleteExpiredSessions(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Sessions $this->deleteByGroup('sessions', [ Query::lessThan('expire', $datetime) @@ -341,8 +343,8 @@ class DeletesV1 extends Worker */ protected function deleteRealtimeUsage(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Dead Realtime Logs $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime) @@ -360,8 +362,9 @@ class DeletesV1 extends Worker throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($datetime); @@ -381,8 +384,9 @@ class DeletesV1 extends Worker throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $audit = new Audit($dbForProject); $status = $audit->cleanup($datetime); if (!$status) { @@ -393,11 +397,11 @@ class DeletesV1 extends Worker /** * @param string $resource - * @param string $projectId + * @param Document $project */ - protected function deleteAuditLogsByResource(string $resource, string $projectId): void + protected function deleteAuditLogsByResource(string $resource, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $this->deleteByGroup(Audit::COLLECTION, [ Query::equal('resource', [$resource]) @@ -406,11 +410,12 @@ class DeletesV1 extends Worker /** * @param Document $document function document - * @param string $projectId + * @param Document $project */ - protected function deleteFunction(Document $document, string $projectId): void + protected function deleteFunction(Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $functionId = $document->getId(); /** @@ -479,11 +484,12 @@ class DeletesV1 extends Worker /** * @param Document $document deployment document - * @param string $projectId + * @param Document $project */ - protected function deleteDeployment(Document $document, string $projectId): void + protected function deleteDeployment(Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $deploymentId = $document->getId(); $functionId = $document->getAttribute('resourceId'); @@ -568,13 +574,11 @@ class DeletesV1 extends Worker $chunk++; /** @var string[] $projectIds */ - $projectIds = array_map(fn (Document $project) => $project->getId(), $projects); - $sum = count($projects); Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects'); - foreach ($projectIds as $projectId) { - $callback($projectId); + foreach ($projects as $project) { + $callback($project); $count++; } } @@ -666,9 +670,10 @@ class DeletesV1 extends Worker } } - protected function deleteBucket(Document $document, string $projectId) + protected function deleteBucket(Document $document, Document $project) { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); $device = $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); diff --git a/app/workers/functions.php b/app/workers/functions.php index 1df776383f..1ba4b6575b 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -52,7 +52,7 @@ class FunctionsV1 extends Worker return; } - $database = $this->getProjectDB($project->getId()); + $database = $this->getProjectDB($project); /** * Handle Event execution. diff --git a/bin/volume-sync b/bin/volume-sync new file mode 100644 index 0000000000..5190750a24 --- /dev/null +++ b/bin/volume-sync @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php volume-sync $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 72b11bcc58..40e17b5fe6 100644 --- a/composer.json +++ b/composer.json @@ -43,25 +43,26 @@ "ext-sockets": "*", "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", + "utopia-php/platform": "0.3.*", "utopia-php/framework": "0.22.*", "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.16.*", "utopia-php/analytics": "0.2.*", - "utopia-php/platform": "0.3.*", - "utopia-php/cli": "0.14.*", - "utopia-php/audit": "0.17.*", "utopia-php/cache": "0.8.*", + "utopia-php/audit": "0.17.*", + "utopia-php/cli": "0.14.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.28.*", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", - "utopia-php/swoole": "0.4.*", + "utopia-php/swoole": "0.5.*", "utopia-php/storage": "0.11.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.7.*", + "utopia-php/pools": "0.4.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index cdcc5886ed..a816d587b6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c0cc0e055f9aa5dc0f3fb4afa9f3fb3d", + "content-hash": "87af69017761fb788e8be70ab5c37bba", "packages": [ { "name": "adhocore/jwt", @@ -115,15 +115,15 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.11.0", + "version": "0.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "547fc026e11c0946846a8ac690898f5bf53be101" + "reference": "9d74a477ba3333cbcfac565c46fcf19606b7b603" }, "require": { "php": ">=8.0", - "utopia-php/system": "0.4.*" + "utopia-php/system": "0.6.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -154,7 +154,7 @@ "php", "runtimes" ], - "time": "2022-08-15T14:03:36+00:00" + "time": "2022-11-07T16:45:52+00:00" }, { "name": "chillerlan/php-qrcode", @@ -300,16 +300,16 @@ }, { "name": "colinmollenhour/credis", - "version": "v1.13.1", + "version": "v1.14.0", "source": { "type": "git", "url": "https://github.com/colinmollenhour/credis.git", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49" + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49", - "reference": "85df015088e00daf8ce395189de22c8eb45c8d49", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/dccc8a46586475075fbb012d8bd523b8a938c2dc", + "reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc", "shasum": "" }, "require": { @@ -341,9 +341,9 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1" + "source": "https://github.com/colinmollenhour/credis/tree/v1.14.0" }, - "time": "2022-06-20T22:56:59+00:00" + "time": "2022-11-09T01:18:39+00:00" }, { "name": "dragonmantank/cron-expression", @@ -803,6 +803,72 @@ }, "time": "2020-12-26T17:45:17+00:00" }, + { + "name": "laravel/pint", + "version": "v1.2.0", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", + "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.11.0", + "illuminate/view": "^9.27", + "laravel-zero/framework": "^9.1.3", + "mockery/mockery": "^1.5.0", + "nunomaduro/larastan": "^2.2", + "nunomaduro/termwind": "^1.14.0", + "pestphp/pest": "^1.22.1" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2022-09-13T15:07:15+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -2193,6 +2259,61 @@ }, "time": "2022-10-28T07:26:09+00:00" }, + { + "name": "utopia-php/pools", + "version": "0.4.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/pools.git", + "reference": "63cf2c32f59675ce9b31619ddd618d0b217e423f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/63cf2c32f59675ce9b31619ddd618d0b217e423f", + "reference": "63cf2c32f59675ce9b31619ddd618d0b217e423f", + "shasum": "" + }, + "require": { + "ext-mongodb": "*", + "ext-pdo": "*", + "ext-redis": "*", + "php": ">=8.0", + "utopia-php/cli": "^0.14.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.8.*", + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Pools\\": "src/Pools" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "description": "A simple library to manage connection pools", + "keywords": [ + "framework", + "php", + "pools", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/pools/issues", + "source": "https://github.com/utopia-php/pools/tree/0.4.0" + }, + "time": "2022-11-10T09:38:57+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -2355,16 +2476,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.4.0", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "536e1f3e78fc0197e4a8ed81b1bf2636a3bc4538" + "reference": "c2a3a4f944a2f22945af3cbcb95b13f0769628b1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/536e1f3e78fc0197e4a8ed81b1bf2636a3bc4538", - "reference": "536e1f3e78fc0197e4a8ed81b1bf2636a3bc4538", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/c2a3a4f944a2f22945af3cbcb95b13f0769628b1", + "reference": "c2a3a4f944a2f22945af3cbcb95b13f0769628b1", "shasum": "" }, "require": { @@ -2373,6 +2494,7 @@ "utopia-php/framework": "0.*.*" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "swoole/ide-helper": "4.8.3", "vimeo/psalm": "4.15.0" @@ -2387,12 +2509,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "team@appwrite.io" - } - ], "description": "An extension for Utopia Framework to work with PHP Swoole as a PHP FPM alternative", "keywords": [ "framework", @@ -2405,29 +2521,31 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.4.0" + "source": "https://github.com/utopia-php/swoole/tree/0.5.0" }, - "time": "2022-10-08T14:32:43+00:00" + "time": "2022-10-19T22:19:07+00:00" }, { "name": "utopia-php/system", - "version": "0.4.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0" + "reference": "289c4327713deadc9c748b5317d248133a02f245" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0", - "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0", + "url": "https://api.github.com/repos/utopia-php/system/zipball/289c4327713deadc9c748b5317d248133a02f245", + "reference": "289c4327713deadc9c748b5317d248133a02f245", "shasum": "" }, "require": { + "laravel/pint": "1.2.*", "php": ">=7.4" }, "require-dev": { "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", "vimeo/psalm": "4.0.1" }, "type": "library", @@ -2460,9 +2578,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.4.0" + "source": "https://github.com/utopia-php/system/tree/0.6.0" }, - "time": "2021-02-04T14:14:49+00:00" + "time": "2022-11-07T13:51:59+00:00" }, { "name": "utopia-php/websocket", @@ -2886,16 +3004,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.1", + "version": "v4.15.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900" + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", "shasum": "" }, "require": { @@ -2936,9 +3054,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.2" }, - "time": "2022-09-04T07:30:47+00:00" + "time": "2022-11-12T15:38:23+00:00" }, { "name": "phar-io/manifest", @@ -4768,16 +4886,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a", + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a", "shasum": "" }, "require": { @@ -4792,7 +4910,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -4830,7 +4948,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0" }, "funding": [ { @@ -4846,20 +4964,20 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534", + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534", "shasum": "" }, "require": { @@ -4874,7 +4992,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -4913,7 +5031,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0" }, "funding": [ { @@ -4929,7 +5047,7 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "textalk/websocket", diff --git a/docker-compose.yml b/docker-compose.yml index ea9241a9d3..16626f7bc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,15 +109,20 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_PUBSUB - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -210,13 +215,19 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_PUBSUB - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -237,15 +248,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -271,6 +286,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -295,15 +311,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -340,22 +360,25 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis - mariadb environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -377,15 +400,19 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -410,15 +437,19 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -439,15 +470,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -539,6 +574,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -565,6 +601,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMS_PROVIDER - _APP_SMS_FROM - _APP_LOGGING_PROVIDER @@ -580,7 +617,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis environment: @@ -588,21 +624,37 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-volume-sync: + entrypoint: volume-sync + <<: *x-logging + container_name: appwrite-volume-sync + image: appwrite-dev + command: + - --source=/data/src/ --destination=/data/dest/ --interval=10 + networks: + - appwrite + # volumes: # Mount the rsync source and destination directories + # - /nfs/config:/data/src + # - /storage/config:/data/dest + appwrite-usage-timeseries: entrypoint: - usage @@ -627,14 +679,17 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -662,14 +717,17 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -691,10 +749,11 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p - container_name: appwrite-mariadb + container_name: mariadb <<: *x-logging networks: - appwrite @@ -713,7 +772,6 @@ services: # smtp: # image: appwrite/smtp:1.2.0 # container_name: appwrite-smtp - # restart: unless-stopped # networks: # - appwrite # environment: @@ -800,7 +858,6 @@ services: image: adminer container_name: appwrite-adminer <<: *x-logging - restart: always ports: - 9506:8080 networks: @@ -808,7 +865,6 @@ services: # redis-commander: # image: rediscommander/redis-commander:latest - # restart: unless-stopped # networks: # - appwrite # environment: @@ -818,7 +874,6 @@ services: # resque: # image: appwrite/resque-web:1.1.0 - # restart: unless-stopped # networks: # - appwrite # ports: @@ -832,7 +887,6 @@ services: # chronograf: # image: chronograf:1.6 # container_name: appwrite-chronograf - # restart: unless-stopped # networks: # - appwrite # volumes: diff --git a/docs/references/health/get-cache.md b/docs/references/health/get-cache.md index 91abcd6bc5..632c02208d 100644 --- a/docs/references/health/get-cache.md +++ b/docs/references/health/get-cache.md @@ -1 +1 @@ -Check the Appwrite in-memory cache server is up and connection is successful. \ No newline at end of file +Check the Appwrite in-memory cache servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-db.md b/docs/references/health/get-db.md index 9652d0d3e3..7381e51f70 100644 --- a/docs/references/health/get-db.md +++ b/docs/references/health/get-db.md @@ -1 +1 @@ -Check the Appwrite database server is up and connection is successful. \ No newline at end of file +Check the Appwrite database servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-pubsub.md b/docs/references/health/get-pubsub.md new file mode 100644 index 0000000000..8f86411e8f --- /dev/null +++ b/docs/references/health/get-pubsub.md @@ -0,0 +1 @@ +Check the Appwrite pub-sub servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-queue.md b/docs/references/health/get-queue.md new file mode 100644 index 0000000000..e4558f941f --- /dev/null +++ b/docs/references/health/get-queue.md @@ -0,0 +1 @@ +Check the Appwrite queue messaging servers are up and connection is successful. \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index 4012c8c276..927e91567a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertWarningsToExceptions="true" processIsolation="false" stopOnFailure="false" -> + > diff --git a/src/Appwrite/CLI/Tasks/Doctor.php b/src/Appwrite/CLI/Tasks/Doctor.php index 2b728b47c8..fbbc5e0230 100644 --- a/src/Appwrite/CLI/Tasks/Doctor.php +++ b/src/Appwrite/CLI/Tasks/Doctor.php @@ -8,6 +8,7 @@ use Appwrite\ClamAV\Network; use Utopia\Logger\Logger; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\Config\Config; use Utopia\Domains\Domain; use Utopia\Platform\Action; use Utopia\Registry\Registry; @@ -36,7 +37,7 @@ class Doctor extends Action Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n"); - Console::log('Checking for production best practices...'); + Console::log('[Settings]'); $domain = new Domain(App::getEnv('_APP_DOMAIN')); @@ -92,7 +93,6 @@ class Doctor extends Action Console::log('🟢 HTTPS force option is enabled'); } - $providerName = App::getEnv('_APP_LOGGING_PROVIDER', ''); $providerConfig = App::getEnv('_APP_LOGGING_CONFIG', ''); @@ -105,30 +105,55 @@ class Doctor extends Action \sleep(0.2); try { - Console::log("\n" . 'Checking connectivity...'); + Console::log("\n" . '[Connectivity]'); } catch (\Throwable $th) { //throw $th; } - try { - $register->get('db'); /* @var $db PDO */ - Console::success('Database............connected 👍'); - } catch (\Throwable $th) { - Console::error('Database.........disconnected 👎'); + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$database})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$database})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}.({$database})", 47, '.') . 'disconnected'); + } + } } - try { - $register->get('cache'); - Console::success('Queue...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Queue............disconnected 👎'); - } + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $configs = [ + 'Cache' => Config::getParam('pools-cache'), + 'Queue' => Config::getParam('pools-queue'), + 'PubSub' => Config::getParam('pools-pubsub'), + ]; - try { - $register->get('cache'); - Console::success('Cache...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Cache............disconnected 👎'); + foreach ($configs as $key => $config) { + foreach ($config as $pool) { + try { + $adapter = $pools->get($pool)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$pool})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } } if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled @@ -139,12 +164,12 @@ class Doctor extends Action ); if ((@$antivirus->ping())) { - Console::success('Antivirus...........connected 👍'); + Console::success('🟢 ' . str_pad("Antivirus", 50, '.') . 'connected'); } else { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } catch (\Throwable $th) { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } @@ -157,35 +182,35 @@ class Doctor extends Action $mail->AltBody = 'Hello World'; $mail->send(); - Console::success('SMTP................connected 👍'); + Console::success('🟢 ' . str_pad("SMTP", 50, '.') . 'connected'); } catch (\Throwable $th) { - Console::error('SMTP.............disconnected 👎'); + Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) { - Console::success('StatsD..............connected 👍'); + Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('StatsD...........disconnected 👎'); + Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_INFLUXDB_HOST', ''); $port = App::getEnv('_APP_INFLUXDB_PORT', ''); if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) { - Console::success('InfluxDB............connected 👍'); + Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('InfluxDB.........disconnected 👎'); + Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected'); } \sleep(0.2); Console::log(''); - Console::log('Checking volumes...'); + Console::log('[Volumes]'); foreach ( [ @@ -213,7 +238,7 @@ class Doctor extends Action \sleep(0.2); Console::log(''); - Console::log('Checking disk space usage...'); + Console::log('[Disk Space]'); foreach ( [ diff --git a/src/Appwrite/CLI/Tasks/Maintenance.php b/src/Appwrite/CLI/Tasks/Maintenance.php index 85973e6500..01b950dad8 100644 --- a/src/Appwrite/CLI/Tasks/Maintenance.php +++ b/src/Appwrite/CLI/Tasks/Maintenance.php @@ -6,13 +6,10 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Utopia\App; -use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; -use Utopia\Database\DateTime; -use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Database\Document; +use Utopia\Database\DateTime; use Utopia\Database\Query; use Utopia\Platform\Action; diff --git a/src/Appwrite/CLI/Tasks/Migrate.php b/src/Appwrite/CLI/Tasks/Migrate.php index 7e36247a55..49878bc6f5 100644 --- a/src/Appwrite/CLI/Tasks/Migrate.php +++ b/src/Appwrite/CLI/Tasks/Migrate.php @@ -8,9 +8,6 @@ use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Database; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; use Utopia\Validator\Text; @@ -44,17 +41,13 @@ class Migrate extends Action Console::success('Starting Data Migration to version ' . $version); - $db = $register->get('db', true); + $dbPool = $register->get('dbPool', true); $redis = $register->get('cache', true); $redis->flushAll(); $cache = new Cache(new RedisCache($redis)); - $projectDB = new Database(new MariaDB($db), $cache); - $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - - $consoleDB = new Database(new MariaDB($db), $cache); - $consoleDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $consoleDB->setNamespace('_project_console'); + $dbForConsole = $dbPool->getDB('console', $cache); + $dbForConsole->setNamespace('_project_console'); $console = $app->getResource('console'); @@ -68,10 +61,10 @@ class Migrate extends Action $count = 0; try { - $totalProjects = $consoleDB->count('projects') + 1; + $totalProjects = $dbForConsole->count('projects') + 1; } catch (\Throwable $th) { - $consoleDB->setNamespace('_console'); - $totalProjects = $consoleDB->count('projects') + 1; + $dbForConsole->setNamespace('_console'); + $totalProjects = $dbForConsole->count('projects') + 1; } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; @@ -87,8 +80,10 @@ class Migrate extends Action } try { + // TODO: Iterate through all project DBs + $projectDB = $dbPool->getDB($project->getId(), $cache); $migration - ->setProject($project, $projectDB, $consoleDB) + ->setProject($project, $projectDB, $dbForConsole) ->execute(); } catch (\Throwable $th) { throw $th; @@ -97,7 +92,7 @@ class Migrate extends Action } $sum = \count($projects); - $projects = $consoleDB->find('projects', [Query::limit($limit), Query::offset($offset)]); + $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); $offset = $offset + $limit; $count = $count + $sum; diff --git a/src/Appwrite/CLI/Tasks/Specs.php b/src/Appwrite/CLI/Tasks/Specs.php index dab6827b33..7c2d242786 100644 --- a/src/Appwrite/CLI/Tasks/Specs.php +++ b/src/Appwrite/CLI/Tasks/Specs.php @@ -8,10 +8,15 @@ use Appwrite\Specification\Format\OpenAPI3; use Appwrite\Specification\Format\Swagger2; use Appwrite\Specification\Specification; use Appwrite\Utopia\Response; +use Exception; use Swoole\Http\Response as HttpResponse; use Utopia\App; +use Utopia\Cache\Adapter\None; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Database; use Utopia\Registry\Registry; use Utopia\Request; use Utopia\Validator\WhiteList; @@ -41,10 +46,11 @@ class Specs extends Action $response = new Response(new HttpResponse()); $mocks = ($mode === 'mocks'); + // Mock dependencies App::setResource('request', fn () => new Request()); App::setResource('response', fn () => $response); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); + App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None()))); + App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None()))); $platforms = [ 'client' => APP_PLATFORM_CLIENT, @@ -219,7 +225,7 @@ class Specs extends Action unset($models[$key]); } } - // var_dump($models); + $arguments = [new App('UTC'), $services, $routes, $models, $keys[$platform], $authCounts[$platform] ?? 0]; foreach (['swagger2', 'open-api3'] as $format) { $formatInstance = match ($format) { diff --git a/src/Appwrite/CLI/Tasks/volume-sync.php b/src/Appwrite/CLI/Tasks/volume-sync.php new file mode 100644 index 0000000000..8d5fec201b --- /dev/null +++ b/src/Appwrite/CLI/Tasks/volume-sync.php @@ -0,0 +1,45 @@ +task('volume-sync') + ->desc('Runs rsync to sync certificates between the storage mount and traefik.') + ->param('source', null, new Text(255), 'Source path to sync from.', false) + ->param('destination', null, new Text(255), 'Destination path to sync to.', false) + ->param('interval', null, new Integer(true), 'Interval to run rsync', false) + ->action(function ($source, $destination, $interval) { + + Console::title('RSync V1'); + Console::success(APP_NAME . ' rsync process v1 has started'); + + if (!file_exists($source)) { + Console::error('Source directory does not exist. Exiting ... '); + Console::exit(0); + } + + Console::loop(function () use ($interval, $source, $destination) { + $time = DateTime::now(); + + Console::info("[{$time}] Executing rsync every {$interval} seconds"); + Console::info("Syncing between $source and $destination"); + + if (!file_exists($source)) { + Console::error('Source directory does not exist. Skipping ... '); + return; + } + + $stdin = ""; + $stdout = ""; + $stderr = ""; + + Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr); + Console::success($stdout); + Console::error($stderr); + }, $interval); + }); diff --git a/src/Appwrite/Extend/PDO.php b/src/Appwrite/Extend/PDO.php deleted file mode 100644 index bc3f42afb9..0000000000 --- a/src/Appwrite/Extend/PDO.php +++ /dev/null @@ -1,110 +0,0 @@ -dsn = $dsn; - $this->username = $username; - $this->passwd = $passwd; - $this->options = $options; - - $this->pdo = new PDONative($dsn, $username, $passwd, $options); - } - - public function setAttribute($attribute, $value) - { - return $this->pdo->setAttribute($attribute, $value); - } - - public function prepare($statement, $driver_options = null) - { - return new PDOStatement($this, $this->pdo->prepare($statement, [])); - } - - public function quote($string, $parameter_type = PDONative::PARAM_STR) - { - return $this->pdo->quote($string, $parameter_type); - } - - public function beginTransaction() - { - try { - $result = $this->pdo->beginTransaction(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->beginTransaction(); - } - - return $result; - } - - public function rollBack() - { - try { - $result = $this->pdo->rollBack(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - return false; - } - - return $result; - } - - public function commit() - { - try { - $result = $this->pdo->commit(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->commit(); - } - - return $result; - } - - public function reconnect(): PDONative - { - $this->pdo = new PDONative($this->dsn, $this->username, $this->passwd, $this->options); - - echo '[PDO] MySQL connection restarted' . PHP_EOL; - - // Connection settings - $this->pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays - $this->pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions - - return $this->pdo; - } -} diff --git a/src/Appwrite/Extend/PDOStatement.php b/src/Appwrite/Extend/PDOStatement.php deleted file mode 100644 index 9c5a83ec34..0000000000 --- a/src/Appwrite/Extend/PDOStatement.php +++ /dev/null @@ -1,115 +0,0 @@ -pdo = &$pdo; - $this->PDOStatement = $PDOStatement; - } - - public function bindValue($parameter, $value, $data_type = PDONative::PARAM_STR) - { - $this->values[$parameter] = ['value' => $value, 'data_type' => $data_type]; - - $result = $this->PDOStatement->bindValue($parameter, $value, $data_type); - - return $result; - } - - public function bindParam($parameter, &$variable, $data_type = PDONative::PARAM_STR, $length = null, $driver_options = null) - { - $this->params[$parameter] = ['value' => &$variable, 'data_type' => $data_type, 'length' => $length, 'driver_options' => $driver_options]; - - $result = $this->PDOStatement->bindParam($parameter, $variable, $data_type, $length, $driver_options); - - return $result; - } - - public function bindColumn($column, &$param, $type = null, $maxlen = null, $driverdata = null) - { - $this->columns[$column] = ['param' => &$param, 'type' => $type, 'maxlen' => $maxlen, 'driverdata' => $driverdata]; - - $result = $this->PDOStatement->bindColumn($column, $param, $type, $maxlen, $driverdata); - - return $result; - } - - public function execute($input_parameters = null) - { - try { - $result = $this->PDOStatement->execute($input_parameters); - } catch (\Throwable $th) { - $this->pdo = $this->pdo->reconnect(); - $this->PDOStatement = $this->pdo->prepare($this->PDOStatement->queryString, []); - - foreach ($this->values as $key => $set) { - $this->PDOStatement->bindValue($key, $set['value'], $set['data_type']); - } - - foreach ($this->params as $key => $set) { - $this->PDOStatement->bindParam($key, $set['variable'], $set['data_type'], $set['length'], $set['driver_options']); - } - - foreach ($this->columns as $key => $set) { - $this->PDOStatement->bindColumn($key, $set['param'], $set['type'], $set['maxlen'], $set['driverdata']); - } - - $result = $this->PDOStatement->execute($input_parameters); - } - - return $result; - } - - public function fetch($fetch_style = PDONative::FETCH_ASSOC, $cursor_orientation = PDONative::FETCH_ORI_NEXT, $cursor_offset = 0) - { - $result = $this->PDOStatement->fetch($fetch_style, $cursor_orientation, $cursor_offset); - - return $result; - } - - /** - * Fetch All - * - * @param int $fetch_style - * @param mixed $fetch_args - * - * @return array|false - */ - public function fetchAll(int $fetch_style = PDO::FETCH_BOTH, mixed ...$fetch_args) - { - $result = $this->PDOStatement->fetchAll(); - - return $result; - } -} diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index 9151a5c0b5..dddbc59e7f 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -321,7 +321,7 @@ class Realtime extends Adapter } } elseif ($parts[2] === 'deployments') { $channels[] = 'console'; - + $projectId = 'console'; $roles = [Role::team($project->getAttribute('teamId'))->toString()]; } diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index c87c907bf0..2ac2641b95 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -86,8 +86,6 @@ abstract class Migration { $this->project = $project; $this->projectDB = $projectDB; - $this->projectDB->setNamespace('_' . $this->project->getId()); - $this->consoleDB = $consoleDB; return $this; diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index dd7cebd084..5d05d77576 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -2,12 +2,12 @@ namespace Appwrite\Resque; +use Exception; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\CLI\Console; +use Utopia\Config\Config; +use Utopia\Cache\Adapter\Sharding; use Utopia\Database\Database; -use Utopia\Database\Adapter\MariaDB; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; @@ -16,7 +16,7 @@ use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; -use Exception; +use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; abstract class Worker @@ -136,7 +136,12 @@ abstract class Worker */ public function tearDown(): void { + global $register; + try { + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $pools->reclaim(); + $this->shutdown(); } catch (\Throwable $error) { foreach (self::$errorCallbacks as $errorCallback) { @@ -158,23 +163,32 @@ abstract class Worker { \array_push(self::$errorCallbacks, $callback); } + /** * Get internal project database - * @param string $projectId + * @param Document $project * @return Database */ - protected function getProjectDB(string $projectId): Database + protected function getProjectDB(Document $project): Database { - $consoleDB = $this->getConsoleDB(); + global $register; - if ($projectId === 'console') { - return $consoleDB; + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return $this->getConsoleDB(); } - /** @var Document $project */ - $project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId)); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; - return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId()); + $database = new Database($dbAdapter, $this->getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; } /** @@ -183,67 +197,46 @@ abstract class Worker */ protected function getConsoleDB(): Database { - return $this->getDB(self::DATABASE_CONSOLE); + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $this->getCache()); + + $database->setNamespace('console'); + + return $database; } + /** - * Get console database - * @param string $type One of (internal, external, console) - * @param string $projectId of internal or external DB - * @return Database + * Get Cache + * @return Cache */ - private function getDB(string $type, string $projectId = '', string $projectInternalId = ''): Database + protected function getCache(): Cache { global $register; - $namespace = ''; - $sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ - switch ($type) { - case self::DATABASE_PROJECT: - if (!$projectId) { - throw new \Exception('ProjectID not provided - cannot get database'); - } - $namespace = "_{$projectInternalId}"; - break; - case self::DATABASE_CONSOLE: - $namespace = "_console"; - $sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created - break; - default: - throw new \Exception('Unknown database type: ' . $type); - break; + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; } - $attempts = 0; - - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($register->get('cache'))); - $database = new Database(new MariaDB($register->get('db')), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); // Main DB - - if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) { - throw new \Exception("Project does not exist: {$projectId}"); - } - - if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; + return new Cache(new Sharding($adapters)); } /** @@ -266,7 +259,6 @@ abstract class Worker return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); } - /** * Get Builds Storage Device * @param string $projectId of the project diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index e23335365a..07134c6ea9 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -207,6 +207,7 @@ class Response extends SwooleResponse public const MODEL_HEALTH_QUEUE = 'healthQueue'; public const MODEL_HEALTH_TIME = 'healthTime'; public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus'; + public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList'; // Deprecated public const MODEL_PERMISSIONS = 'permissions'; @@ -268,6 +269,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) ->setModel(new BaseList('Variables List', self::MODEL_VARIABLE_LIST, 'variables', self::MODEL_VARIABLE)) + ->setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS)) // Entities ->setModel(new Database()) ->setModel(new Collection()) diff --git a/src/Appwrite/Utopia/Response/Model/HealthStatus.php b/src/Appwrite/Utopia/Response/Model/HealthStatus.php index 23756de131..ba340107ac 100644 --- a/src/Appwrite/Utopia/Response/Model/HealthStatus.php +++ b/src/Appwrite/Utopia/Response/Model/HealthStatus.php @@ -10,6 +10,12 @@ class HealthStatus extends Model public function __construct() { $this + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Name of the service.', + 'default' => '', + 'example' => 'database', + ]) ->addRule('ping', [ 'type' => self::TYPE_INTEGER, 'description' => 'Duration in milliseconds how long the health check took.', diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 47a2268e21..96c9bde5c7 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -47,9 +47,9 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE @@ -69,9 +69,53 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testQueueSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testPubSubSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/pubsub', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE diff --git a/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php b/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php index efcfe95f53..500b3d48be 100644 --- a/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php @@ -2,16 +2,19 @@ namespace Tests\E2E\Services\Realtime; +use CURLFile; use Tests\E2E\Client; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\SideConsole; +use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Database\ID; use Utopia\Database\Permission; use Utopia\Database\Role; class RealtimeConsoleClientTest extends Scope { + use FunctionsBase; use RealtimeBase; use ProjectCustom; use SideConsole; @@ -425,4 +428,78 @@ class RealtimeConsoleClientTest extends Scope $client->close(); } + + public function testCreateDeployment() + { + $response1 = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test', + 'runtime' => 'php-8.0', + 'events' => [ + 'users.*.create', + 'users.*.delete', + ], + 'schedule' => '0 0 1 1 *', + 'timeout' => 10, + ]); + + $functionId = $response1['body']['$id'] ?? ''; + + $this->assertEquals(201, $response1['headers']['status-code']); + + + $projectId = 'console'; + + $client = $this->getWebsocket(['console'], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ], $projectId); + + $response = json_decode($client->receive(), true); + + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('connected', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertCount(1, $response['data']['channels']); + $this->assertContains('console', $response['data']['channels']); + $this->assertNotEmpty($response['data']['user']); + + /** + * Test Create Deployment + */ + + $folder = 'php'; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/$folder/code.tar.gz"; + $this->packageCode($folder); + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', \basename($code)), + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + + $this->assertEquals(202, $deployment['headers']['status-code']); + + $response = json_decode($client->receive(), true); + + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(1, $response['data']['channels']); + $this->assertContains('console', $response['data']['channels']); + $this->assertContains("functions.{$functionId}.deployments.{$deploymentId}.create", $response['data']['events']); + $this->assertNotEmpty($response['data']['payload']); + + $client->close(); + } }