From c6c7734c29e8a8bd0873054ac191cd2b5b77ec9a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 11 Oct 2023 22:01:01 +0200 Subject: [PATCH 001/406] Merge pull request #6496 from appwrite/fix-readme-cn Update logo in README-CN.md ## What does this PR do? (Provide a description of what this PR does and why it's needed.) ## Test Plan (Write your test plan here. If you changed any code, please provide us with clear instructions on how you verified your changes work. Screenshots may also be helpful.) ## Related PRs and Issues - (Related PR or issue) ## Checklist - [ ] Have you read the [Contributing Guidelines on issues](https://github.com/appwrite/appwrite/blob/master/CONTRIBUTING.md)? - [ ] If the PR includes a change to an API's metadata (desc, label, params, etc.), does it also include updated API specs and example docs? --- app/http.php | 23 +++++- app/init.php | 76 +++++++++++-------- app/test | 184 ++++++++++++++++++++++++++++++++++++++++++++++ composer.lock | 22 +++--- package 2.json | 8 ++ package-lock.json | 10 +++ pnpm-lock.yaml | 5 ++ 7 files changed, 284 insertions(+), 44 deletions(-) create mode 100644 app/test create mode 100644 package 2.json create mode 100644 package-lock.json create mode 100644 pnpm-lock.yaml diff --git a/app/http.php b/app/http.php index fe1ed48724..72b89130b9 100644 --- a/app/http.php +++ b/app/http.php @@ -328,7 +328,28 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - $pools->reclaim(); + $connectionForConsole = $app->getResource('connectionForConsole'); + $connectionForProject = $app->getResource('connectionForProject'); + $connectionForQueue = $app->getResource('connectionForQueue'); + $connectionsForCache = $app->getResource('connectionsForCache'); + + if(!is_null($connectionForConsole)) { + $connectionForConsole->reclaim(); + } + + if(!is_null($connectionForProject)) { + $connectionForProject->reclaim(); + } + + if(!is_null($connectionForQueue)) { + $connectionForQueue->reclaim(); + } + + if(!empty($connectionsForCache)) { + foreach ($connectionsForCache as $connection) { + $connection->reclaim(); + } + } } }); diff --git a/app/init.php b/app/init.php index ba133ec7fb..7078f90870 100644 --- a/app/init.php +++ b/app/init.php @@ -893,12 +893,6 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); -App::setResource('queue', function (Group $pools) { - return $pools->get('queue')->pop()->getResource(); -}, ['pools']); -App::setResource('queueForFunctions', function (Connection $queue) { - return new Func($queue); -}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); @@ -1091,17 +1085,25 @@ App::setResource('console', function () { ]); }, []); +App::setResource('connectionForProject', function () { return null; }, []); +App::setResource('connectionForConsole', function () { return null; }, []); +App::setResource('connectionForQueue', function () { return null; }, []); +App::setResource('connectionsForCache', function () { return []; }, []); + App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } - $dbAdapter = $pools + $connection = $pools ->get($project->getAttribute('database')) ->pop() - ->getResource() ; + App::setResource('connectionForProject', function () use ($connection) { return $connection; }, []); + + $dbAdapter = $connection->getResource(); + $database = new Database($dbAdapter, $cache); $database->setNamespace('_' . $project->getInternalId()); @@ -1109,11 +1111,13 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, }, ['pools', 'dbForConsole', 'cache', 'project']); App::setResource('dbForConsole', function (Group $pools, Cache $cache) { - $dbAdapter = $pools + $connection = $pools ->get('console') - ->pop() - ->getResource() - ; + ->pop(); + + App::setResource('connectionForConsole', function () use ($connection) { return $connection; }, []); + + $dbAdapter = $connection->getResource(); $database = new Database($dbAdapter, $cache); @@ -1123,30 +1127,22 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { }, ['pools', 'cache']); App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { - $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } - - $databaseName = $project->getAttribute('database'); - - if (isset($databases[$databaseName])) { - $database = $databases[$databaseName]; - $database->setNamespace('_' . $project->getInternalId()); - return $database; - } - - $dbAdapter = $pools - ->get($databaseName) + + $connection = $pools + ->get($project->getAttribute('database')) ->pop() - ->getResource(); + ; + + $dbAdapter = $connection->getResource(); + + App::setResource('connectionForProject', function () use ($connection) { return $connection; }, []); $database = new Database($dbAdapter, $cache); - $databases[$databaseName] = $database; - $database->setNamespace('_' . $project->getInternalId()); return $database; @@ -1155,18 +1151,34 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $getProjectDB; }, ['pools', 'dbForConsole', 'cache']); +App::setResource('queue', function (Group $pools) { + $connection = $pools->get('queue')->pop(); + + App::setResource('connectionForQueue', function () use ($connection) { return $connection; }, []); + + return $connection->getResource(); +}, ['pools']); + +App::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); + App::setResource('cache', function (Group $pools) { $list = Config::getParam('pools-cache', []); $adapters = []; + $connections = []; foreach ($list as $value) { - $adapters[] = $pools + $connection = $pools ->get($value) - ->pop() - ->getResource() - ; + ->pop(); + + $connections[] = $connection; + $adapters[] = $connection->getResource(); } + App::setResource('connectionsForCache', function () use ($connections) { return $connections; }, []); + return new Cache(new Sharding($adapters)); }, ['pools']); diff --git a/app/test b/app/test new file mode 100644 index 0000000000..a4d28b1200 --- /dev/null +++ b/app/test @@ -0,0 +1,184 @@ +$register->set('pools', function () { + $group = new Group(); + + $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => App::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => App::getEnv('_APP_DB_PORT', '3306'), + 'user' => App::getEnv('_APP_DB_USER', ''), + 'pass' => App::getEnv('_APP_DB_PASS', ''), + 'path' => App::getEnv('_APP_DB_SCHEMA', ''), + ]); + $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); + + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; + + $maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151); + $instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14); + + $multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; + + if ($multiprocessing) { + $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); + } else { + $workerCount = 1; + } + + if ($workerCount > $instanceConnections) { + throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); + } + + $poolSize = (int)($instanceConnections / $workerCount); + + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $dsns = $connection['dsns'] ?? ''; + $multipe = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getPath(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused accross connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + switch ($dsnScheme) { + case 'mysql': + case 'mariadb': + $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }; + break; + case 'redis': + $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); + break; + } + + $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { + // Get Adapter + $adapter = null; + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + $adapter->setDefaultDatabase($dsn->getPath()); + break; + case 'pubsub': + $adapter = $resource(); + break; + case 'queue': + $adapter = match ($dsn->getScheme()) { + 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), + default => null + }; + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + break; + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); + } + + return $group; +}); diff --git a/composer.lock b/composer.lock index 288a17101f..27712b38ac 100644 --- a/composer.lock +++ b/composer.lock @@ -1050,16 +1050,16 @@ }, { "name": "matomo/device-detector", - "version": "6.1.5", + "version": "6.1.6", "source": { "type": "git", "url": "https://github.com/matomo-org/device-detector.git", - "reference": "40ca2990dba2c1719e5c62168e822e0b86c167d4" + "reference": "5cbea85106e561c7138d03603eb6e05128480409" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matomo-org/device-detector/zipball/40ca2990dba2c1719e5c62168e822e0b86c167d4", - "reference": "40ca2990dba2c1719e5c62168e822e0b86c167d4", + "url": "https://api.github.com/repos/matomo-org/device-detector/zipball/5cbea85106e561c7138d03603eb6e05128480409", + "reference": "5cbea85106e561c7138d03603eb6e05128480409", "shasum": "" }, "require": { @@ -1115,7 +1115,7 @@ "source": "https://github.com/matomo-org/matomo", "wiki": "https://dev.matomo.org/" }, - "time": "2023-08-17T16:17:41+00:00" + "time": "2023-10-02T10:01:54+00:00" }, { "name": "mongodb/mongodb", @@ -2152,16 +2152,16 @@ }, { "name": "utopia-php/database", - "version": "0.43.4", + "version": "0.43.5", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "cabdd02e8dc1732eb0b22007c511e7bb3caa5c8c" + "reference": "5f7b05189cfbcc0506090498c580c5765375a00a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/cabdd02e8dc1732eb0b22007c511e7bb3caa5c8c", - "reference": "cabdd02e8dc1732eb0b22007c511e7bb3caa5c8c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/5f7b05189cfbcc0506090498c580c5765375a00a", + "reference": "5f7b05189cfbcc0506090498c580c5765375a00a", "shasum": "" }, "require": { @@ -2202,9 +2202,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.43.4" + "source": "https://github.com/utopia-php/database/tree/0.43.5" }, - "time": "2023-09-28T09:00:05+00:00" + "time": "2023-10-06T06:49:47+00:00" }, { "name": "utopia-php/domains", diff --git a/package 2.json b/package 2.json new file mode 100644 index 0000000000..6e32c7d515 --- /dev/null +++ b/package 2.json @@ -0,0 +1,8 @@ +{ + "private": true, + "name": "@appwrite.io/repo", + "repository": { + "type": "git", + "url": "git+https://github.com/appwrite/appwrite.git" + } +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000..6ab33b14fe --- /dev/null +++ b/package-lock.json @@ -0,0 +1,10 @@ +{ + "name": "@appwrite.io/repo", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@appwrite.io/repo" + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000000..2b9f1883a1 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,5 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false From 0ea1418132b194c173e6efe211042e985c73b075 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 21 Oct 2023 06:57:59 -0400 Subject: [PATCH 002/406] Sync with main --- .../examples/health/get-queue-builds.md | 18 - .../examples/health/get-queue-databases.md | 18 - .../examples/health/get-queue-deletes.md | 18 - .../examples/health/get-queue-mails.md | 18 - .../Platform/Workers/Certificates.php | 534 --------------- src/Appwrite/Platform/Workers/Databases.php | 622 ------------------ src/Appwrite/Platform/Workers/Mails.php | 126 ---- src/Appwrite/Platform/Workers/Migrations.php | 330 ---------- src/Appwrite/Platform/Workers/Webhooks.php | 118 ---- 9 files changed, 1802 deletions(-) delete mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md delete mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md delete mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md delete mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md delete mode 100644 src/Appwrite/Platform/Workers/Certificates.php delete mode 100644 src/Appwrite/Platform/Workers/Databases.php delete mode 100644 src/Appwrite/Platform/Workers/Mails.php delete mode 100644 src/Appwrite/Platform/Workers/Migrations.php delete mode 100644 src/Appwrite/Platform/Workers/Webhooks.php diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md deleted file mode 100644 index a312fa7df0..0000000000 --- a/docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md +++ /dev/null @@ -1,18 +0,0 @@ -import { Client, Health } from "@appwrite.io/console"; - -const client = new Client(); - -const health = new Health(client); - -client - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject('5df5acd0d48c2') // Your project ID -; - -const promise = health.getQueueBuilds(); - -promise.then(function (response) { - console.log(response); // Success -}, function (error) { - console.log(error); // Failure -}); \ No newline at end of file diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md deleted file mode 100644 index 481480a80d..0000000000 --- a/docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md +++ /dev/null @@ -1,18 +0,0 @@ -import { Client, Health } from "@appwrite.io/console"; - -const client = new Client(); - -const health = new Health(client); - -client - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject('5df5acd0d48c2') // Your project ID -; - -const promise = health.getQueueDatabases(); - -promise.then(function (response) { - console.log(response); // Success -}, function (error) { - console.log(error); // Failure -}); \ No newline at end of file diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md deleted file mode 100644 index c1bbb9f655..0000000000 --- a/docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md +++ /dev/null @@ -1,18 +0,0 @@ -import { Client, Health } from "@appwrite.io/console"; - -const client = new Client(); - -const health = new Health(client); - -client - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject('5df5acd0d48c2') // Your project ID -; - -const promise = health.getQueueDeletes(); - -promise.then(function (response) { - console.log(response); // Success -}, function (error) { - console.log(error); // Failure -}); \ No newline at end of file diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md deleted file mode 100644 index 8d6d68228a..0000000000 --- a/docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md +++ /dev/null @@ -1,18 +0,0 @@ -import { Client, Health } from "@appwrite.io/console"; - -const client = new Client(); - -const health = new Health(client); - -client - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject('5df5acd0d48c2') // Your project ID -; - -const promise = health.getQueueMails(); - -promise.then(function (response) { - console.log(response); // Success -}, function (error) { - console.log(error); // Failure -}); \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php deleted file mode 100644 index 02c1835dd5..0000000000 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ /dev/null @@ -1,534 +0,0 @@ -desc('Certificates worker') - ->inject('message') - ->inject('dbForConsole') - ->inject('queueForMails') - ->inject('queueForEvents') - ->inject('queueForFunctions') - ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions)); - } - - /** - * @param Message $message - * @param Database $dbForConsole - * @param Mail $queueForMails - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @return void - * @throws Throwable - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void - { - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - $document = new Document($payload['domain'] ?? []); - $domain = new Domain($document->getAttribute('domain', '')); - $skipRenewCheck = $payload['skipRenewCheck'] ?? false; - - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck); - } - - /** - * @param Domain $domain - * @param Database $dbForConsole - * @param Mail $queueForMails - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @param bool $skipRenewCheck - * @return void - * @throws Throwable - * @throws \Utopia\Database\Exception - */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void - { - /** - * 1. Read arguments and validate domain - * 2. Get main domain - * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) - * 4. Validate security email. Cannot be empty, required by LetsEncrypt - * 5. Validate renew date with certificate file, unless requested to skip by parameter - * 6. Issue a certificate using certbot CLI - * 7. Update 'log' attribute on certificate document with Certbot message - * 8. Create storage folder for certificate, if not ready already - * 9. Move certificates from Certbot location to our Storage - * 10. Create/Update our Storage with new Traefik config with new certificate paths - * 11. Read certificate file and update 'renewDate' on certificate document - * 12. Update 'issueDate' and 'attempts' on certificate - * - * If at any point unexpected error occurs, program stops without applying changes to document, and error is thrown into worker - * - * If code stops with expected error: - * 1. 'log' attribute on document is updated with error message - * 2. 'attempts' amount is increased - * 3. Console log is shown - * 4. Email is sent to security email - * - * Unless unexpected error occurs, at the end, we: - * 1. Update 'updated' attribute on document - * 2. Save document to database - * 3. Update all domains documents with current certificate ID - * - * Note: Renewals are checked and scheduled from maintenence worker - */ - - // Get current certificate - $certificate = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain->get()])]); - - // If we don't have certificate for domain yet, let's create new document. At the end we save it - if (!$certificate) { - $certificate = new Document(); - $certificate->setAttribute('domain', $domain->get()); - } - - $success = false; - - try { - // Email for alerts is required by LetsEncrypt - $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); - } - - // Validate domain and DNS records. Skip if job is forced - if (!$skipRenewCheck) { - $mainDomain = $this->getMainDomain(); - $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; - $this->validateDomain($domain, $isMainDomain); - } - - // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { - throw new Exception('Renew isn\'t required.'); - } - - // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate - $folder = ID::unique(); - - // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); - - // Command succeeded, store all data into document - $logs = 'Certificate successfully generated.'; - $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB - - - // Give certificates to Traefik - $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); - - // Update certificate info stored in database - $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); - $certificate->setAttribute('attempts', 0); - $certificate->setAttribute('issueDate', DateTime::now()); - $success = true; - } catch (Throwable $e) { - $logs = $e->getMessage(); - - // Set exception as log in certificate document - $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB - - // Increase attempts count - $attempts = $certificate->getAttribute('attempts', 0) + 1; - $certificate->setAttribute('attempts', $attempts); - - // Store cuttent time as renew date to ensure another attempt in next maintenance cycle - $certificate->setAttribute('renewDate', DateTime::now()); - - // Send email to security email - $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails); - } finally { - // All actions result in new updatedAt date - $certificate->setAttribute('updated', DateTime::now()); - - // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForConsole, $queueForEvents, $queueForFunctions); - } - } - - /** - * Save certificate data into database. - * - * @param string $domain Domain name that certificate is for - * @param Document $certificate Certificate document that we need to save - * @param bool $success - * @param Database $dbForConsole Database connection for console - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @return void - * @throws \Utopia\Database\Exception - * @throws Authorization - * @throws Conflict - * @throws Structure - */ - private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void - { - // Check if update or insert required - $certificateDocument = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]); - if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { - // Merge new data with current data - $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); - $certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); - } else { - $certificate->removeAttribute('$internalId'); - $certificate = $dbForConsole->createDocument('certificates', $certificate); - } - - $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForConsole, $queueForEvents, $queueForFunctions); - } - - /** - * Get main domain. Needed as we do different checks for main and non-main domains. - * - * @return null|string Returns main domain. If null, there is no main domain yet. - */ - private function getMainDomain(): ?string - { - $envDomain = App::getEnv('_APP_DOMAIN', ''); - if (!empty($envDomain) && $envDomain !== 'localhost') { - return $envDomain; - } - - return null; - } - - /** - * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) - * - Domain must have proper DNS record - * - * @param Domain $domain Domain which we validate - * @param bool $isMainDomain In case of master domain, we look for different DNS configurations - * - * @return void - * @throws Exception - */ - private function validateDomain(Domain $domain, bool $isMainDomain): void - { - if (empty($domain->get())) { - throw new Exception('Missing certificate domain.'); - } - - if (!$domain->isKnown() || $domain->isTest()) { - throw new Exception('Unknown public suffix for domain.'); - } - - if (!$isMainDomain) { - // TODO: Would be awesome to also support A/AAAA records here. Maybe dry run? - // Validate if domain target is properly configured - $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); - - if (!$target->isKnown() || $target->isTest()) { - throw new Exception('Unreachable CNAME target (' . $target->get() . '), please use a domain with a public suffix.'); - } - - // Verify domain with DNS records - $validator = new CNAME($target->get()); - if (!$validator->isValid($domain->get())) { - throw new Exception('Failed to verify domain DNS records.'); - } - } else { - // Main domain validation - // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? - } - } - - /** - * Reads expiry date of certificate from file and decides if renewal is required or not. - * - * @param string $domain Domain for which we check certificate file - * @return bool True, if certificate needs to be renewed - * @throws Exception - */ - private function isRenewRequired(string $domain): bool - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - if (\file_exists($certPath)) { - $validTo = null; - - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? 0; - - if (empty($validTo)) { - throw new Exception('Unable to read certificate file (cert.pem).'); - } - - // LetsEncrypt allows renewal 30 days before expiry - $expiryInAdvance = (60 * 60 * 24 * 30); - if ($validTo - $expiryInAdvance > \time()) { - return false; - } - } - - return true; - } - - /** - * LetsEncrypt communication to issue certificate (using certbot CLI) - * - * @param string $folder Folder into which certificates should be generated - * @param string $domain Domain to generate certificate for - * @return array Named array with keys 'stdout' and 'stderr', both string - * @throws Exception - */ - private function issueCertificate(string $folder, string $domain, string $email): array - { - $stdout = ''; - $stderr = ''; - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " --cert-name " . $folder - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain}", '', $stdout, $stderr); - - // Unexpected error, usually 5XX, API limits, ... - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - return [ - 'stdout' => $stdout, - 'stderr' => $stderr - ]; - } - - /** - * Read new renew date from certificate file generated by Let's Encrypt - * - * @param string $domain Domain which certificate was generated for - * @return string - * @throws \Utopia\Database\Exception - */ - private function getRenewDate(string $domain): string - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? null; - $dt = (new \DateTime())->setTimestamp($validTo); - return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days - } - - /** - * Method to take files from Let's Encrypt, and put it into Traefik. - * - * @param string $domain Domain which certificate was generated for - * @param string $folder Folder in which certificates were generated - * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error - * @return void - * @throws Exception - */ - private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void - { - - // Prepare folder in storage for domain - $path = APP_STORAGE_CERTIFICATES . '/' . $domain; - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path for certificate.'); - } - } - - // Move generated files - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - $config = \implode(PHP_EOL, [ - "tls:", - " certificates:", - " - certFile: /storage/certificates/{$domain}/fullchain.pem", - " keyFile: /storage/certificates/{$domain}/privkey.pem" - ]); - - // Save configuration into Traefik using our new cert files - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { - throw new Exception('Failed to save Traefik configuration.'); - } - } - - /** - * Method to make sure information about error is delivered to admnistrator. - * - * @param string $domain Domain that caused the error - * @param string $errorMessage Verbose error message - * @param int $attempt How many times it failed already - * @param Mail $queueForMails - * @return void - * @throws Exception - */ - private function notifyError(string $domain, string $errorMessage, int $attempt, Mail $queueForMails): void - { - // Log error into console - Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage); - - // Send mail to administratore mail - - $locale = new Locale(App::getEnv('_APP_LOCALE', 'en')); - if (!$locale->getText('emails.sender') || !$locale->getText("emails.certificate.hello") || !$locale->getText("emails.certificate.subject") || !$locale->getText("emails.certificate.body") || !$locale->getText("emails.certificate.footer") || !$locale->getText("emails.certificate.thanks") || !$locale->getText("emails.certificate.signature")) { - $locale->setDefault('en'); - } - - $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); - - $subject = \sprintf($locale->getText("emails.certificate.subject"), $domain); - $body - ->setParam('{{domain}}', $domain) - ->setParam('{{error}}', $errorMessage) - ->setParam('{{attempt}}', $attempt) - ->setParam('{{subject}}', $subject) - ->setParam('{{hello}}', $locale->getText("emails.certificate.hello")) - ->setParam('{{body}}', $locale->getText("emails.certificate.body")) - ->setParam('{{redirect}}', 'https://' . $domain) - ->setParam('{{footer}}', $locale->getText("emails.certificate.footer")) - ->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks")) - ->setParam('{{signature}}', $locale->getText("emails.certificate.signature")) - ->setParam('{{project}}', 'Console') - ->setParam('{{direction}}', $locale->getText('settings.direction')) - ->setParam('{{bg-body}}', '#f7f7f7') - ->setParam('{{bg-content}}', '#ffffff') - ->setParam('{{text-content}}', '#000000'); - - $queueForMails - ->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')) - ->setBody($body->render()) - ->setName('Appwrite Administrator') - ->trigger(); - } - - /** - * Update all existing domain documents so they have relation to correct certificate document. - * This solved issues: - * - when adding a domain for which there is already a certificate - * - when renew creates new document? It might? - * - overall makes it more reliable - * - * @param string $certificateId ID of a new or updated certificate document - * @param string $domain Domain that is affected by new certificate - * @param bool $success Was certificate generation successful? - * - * @return void - */ - private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void - { - - $rule = $dbForConsole->findOne('rules', [ - Query::equal('domain', [$domain]), - ]); - - if ($rule !== false && !$rule->isEmpty()) { - $rule->setAttribute('certificateId', $certificateId); - $rule->setAttribute('status', $success ? 'verified' : 'unverified'); - $dbForConsole->updateDocument('rules', $rule->getId(), $rule); - - $projectId = $rule->getAttribute('projectId'); - - // Skip events for console project (triggered by auto-ssl generation for 1 click setups) - if ($projectId === 'console') { - return; - } - - $project = $dbForConsole->getDocument('projects', $projectId); - - /** Trigger Webhook */ - $ruleModel = new Rule(); - $queueForEvents - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) - ->trigger(); - - - /** Trigger Functions */ - $queueForFunctions - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) - ->trigger(); - - /** Trigger realtime event */ - $allEvents = Event::generateEvents('rules.[ruleId].update', [ - 'ruleId' => $rule->getId(), - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $rule, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - } - } -} diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php deleted file mode 100644 index e0ec75e1d4..0000000000 --- a/src/Appwrite/Platform/Workers/Databases.php +++ /dev/null @@ -1,622 +0,0 @@ -desc('Databases worker') - ->inject('message') - ->inject('dbForConsole') - ->inject('dbForProject') - ->callback(fn($message, $dbForConsole, $dbForProject) => $this->action($message, $dbForConsole, $dbForProject)); - } - - /** - * @param Message $message - * @param Database $dbForConsole - * @param Database $dbForProject - * @return void - * @throws \Exception - */ - public function action(Message $message, Database $dbForConsole, Database $dbForProject): void - { - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new \Exception('Missing payload'); - } - - $type = $payload['type']; - $project = new Document($payload['project']); - $collection = new Document($payload['collection'] ?? []); - $document = new Document($payload['document'] ?? []); - $database = new Document($payload['database'] ?? []); - - if ($database->isEmpty()) { - throw new Exception('Missing database'); - } - - match (strval($type)) { - DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), - DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), - DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), - DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), - DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), - DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), - default => Console::error('No database operation for type: ' . $type), - }; - } - - /** - * @param Document $database - * @param Document $collection - * @param Document $attribute - * @param Document $project - * @param Database $dbForConsole - * @param Database $dbForProject - * @return void - * @throws Authorization - * @throws Conflict - * @throws \Exception - */ - private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void - { - if ($collection->isEmpty()) { - throw new Exception('Missing collection'); - } - if ($attribute->isEmpty()) { - throw new Exception('Missing attribute'); - } - - $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'attributeId' => $attribute->getId() - ]); - /** - * TODO @christyjacob4 verify if this is still the case - * Fetch attribute from the database, since with Resque float values are loosing informations. - */ - $attribute = $dbForProject->getDocument('attributes', $attribute->getId()); - - $collectionId = $collection->getId(); - $key = $attribute->getAttribute('key', ''); - $type = $attribute->getAttribute('type', ''); - $size = $attribute->getAttribute('size', 0); - $required = $attribute->getAttribute('required', false); - $default = $attribute->getAttribute('default', null); - $signed = $attribute->getAttribute('signed', true); - $array = $attribute->getAttribute('array', false); - $format = $attribute->getAttribute('format', ''); - $formatOptions = $attribute->getAttribute('formatOptions', []); - $filters = $attribute->getAttribute('filters', []); - $options = $attribute->getAttribute('options', []); - $project = $dbForConsole->getDocument('projects', $projectId); - - - try { - switch ($type) { - case Database::VAR_RELATIONSHIP: - $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']); - if ($relatedCollection->isEmpty()) { - throw new DatabaseException('Collection not found'); - } - - if ( - !$dbForProject->createRelationship( - collection: 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - relatedCollection: 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), - type: $options['relationType'], - twoWay: $options['twoWay'], - id: $key, - twoWayKey: $options['twoWayKey'], - onDelete: $options['onDelete'], - ) - ) { - throw new DatabaseException('Failed to create Attribute'); - } - - if ($options['twoWay']) { - $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']); - $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'available')); - } - break; - default: - if (!$dbForProject->createAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) { - throw new \Exception('Failed to create Attribute'); - } - } - - $dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available')); - } catch (\Exception $e) { - Console::error($e->getMessage()); - - if ($e instanceof DatabaseException) { - $attribute->setAttribute('error', $e->getMessage()); - if (isset($relatedAttribute)) { - $relatedAttribute->setAttribute('error', $e->getMessage()); - } - } - - $dbForProject->updateDocument( - 'attributes', - $attribute->getId(), - $attribute->setAttribute('status', 'failed') - ); - - if (isset($relatedAttribute)) { - $dbForProject->updateDocument( - 'attributes', - $relatedAttribute->getId(), - $relatedAttribute->setAttribute('status', 'failed') - ); - } - } finally { - $this->trigger($database, $collection, $attribute, $project, $projectId, $events); - } - - if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) { - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); - } - - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); - } - - /** - * @param Document $database - * @param Document $collection - * @param Document $attribute - * @param Document $project - * @param Database $dbForConsole - * @param Database $dbForProject - * @return void - * @throws Authorization - * @throws Conflict - * @throws \Exception - **/ - private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void - { - if ($collection->isEmpty()) { - throw new Exception('Missing collection'); - } - if ($attribute->isEmpty()) { - throw new Exception('Missing attribute'); - } - - $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'attributeId' => $attribute->getId() - ]); - $collectionId = $collection->getId(); - $key = $attribute->getAttribute('key', ''); - $status = $attribute->getAttribute('status', ''); - $type = $attribute->getAttribute('type', ''); - $project = $dbForConsole->getDocument('projects', $projectId); - $options = $attribute->getAttribute('options', []); - $relatedAttribute = new Document(); - $relatedCollection = new Document(); - // possible states at this point: - // - available: should not land in queue; controller flips these to 'deleting' - // - processing: hasn't finished creating - // - deleting: was available, in deletion queue for first time - // - failed: attribute was never created - // - stuck: attribute was available but cannot be removed - - try { - if ($status !== 'failed') { - if ($type === Database::VAR_RELATIONSHIP) { - if ($options['twoWay']) { - $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']); - if ($relatedCollection->isEmpty()) { - throw new DatabaseException('Collection not found'); - } - $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']); - } - - if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { - $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck')); - throw new DatabaseException('Failed to delete Relationship'); - } - } elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { - throw new DatabaseException('Failed to delete Attribute'); - } - } - - $dbForProject->deleteDocument('attributes', $attribute->getId()); - - if (!$relatedAttribute->isEmpty()) { - $dbForProject->deleteDocument('attributes', $relatedAttribute->getId()); - } - } catch (\Exception $e) { - Console::error($e->getMessage()); - - if ($e instanceof DatabaseException) { - $attribute->setAttribute('error', $e->getMessage()); - if (!$relatedAttribute->isEmpty()) { - $relatedAttribute->setAttribute('error', $e->getMessage()); - } - } - $dbForProject->updateDocument( - 'attributes', - $attribute->getId(), - $attribute->setAttribute('status', 'stuck') - ); - if (!$relatedAttribute->isEmpty()) { - $dbForProject->updateDocument( - 'attributes', - $relatedAttribute->getId(), - $relatedAttribute->setAttribute('status', 'stuck') - ); - } - } finally { - $this->trigger($database, $collection, $attribute, $project, $projectId, $events); - } - - // The underlying database removes/rebuilds indexes when attribute is removed - // Update indexes table with changes - /** @var Document[] $indexes */ - $indexes = $collection->getAttribute('indexes', []); - - foreach ($indexes as $index) { - /** @var string[] $attributes */ - $attributes = $index->getAttribute('attributes'); - $lengths = $index->getAttribute('lengths'); - $orders = $index->getAttribute('orders'); - - $found = \array_search($key, $attributes); - - if ($found !== false) { - // If found, remove entry from attributes, lengths, and orders - // array_values wraps array_diff to reindex array keys - // when found attribute is removed from array - $attributes = \array_values(\array_diff($attributes, [$attributes[$found]])); - $lengths = \array_values(\array_diff($lengths, isset($lengths[$found]) ? [$lengths[$found]] : [])); - $orders = \array_values(\array_diff($orders, isset($orders[$found]) ? [$orders[$found]] : [])); - - if (empty($attributes)) { - $dbForProject->deleteDocument('indexes', $index->getId()); - } else { - $index - ->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN) - ->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN) - ->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN); - - // Check if an index exists with the same attributes and orders - $exists = false; - foreach ($indexes as $existing) { - if ( - $existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself - && $existing->getAttribute('attributes') === $index->getAttribute('attributes') - && $existing->getAttribute('orders') === $index->getAttribute('orders') - ) { - $exists = true; - break; - } - } - - if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject); - } else { - $dbForProject->updateDocument('indexes', $index->getId(), $index); - } - } - } - } - - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); - $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - - if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) { - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); - } - } - - /** - * @param Document $database - * @param Document $collection - * @param Document $index - * @param Document $project - * @param Database $dbForConsole - * @param Database $dbForProject - * @return void - * @throws Authorization - * @throws Conflict - * @throws Structure - * @throws DatabaseException - */ - private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void - { - if ($collection->isEmpty()) { - throw new Exception('Missing collection'); - } - if ($index->isEmpty()) { - throw new Exception('Missing index'); - } - - $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'indexId' => $index->getId() - ]); - $collectionId = $collection->getId(); - $key = $index->getAttribute('key', ''); - $type = $index->getAttribute('type', ''); - $attributes = $index->getAttribute('attributes', []); - $lengths = $index->getAttribute('lengths', []); - $orders = $index->getAttribute('orders', []); - $project = $dbForConsole->getDocument('projects', $projectId); - - try { - if (!$dbForProject->createIndex('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $attributes, $lengths, $orders)) { - throw new DatabaseException('Failed to create Index'); - } - $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available')); - } catch (\Exception $e) { - Console::error($e->getMessage()); - - if ($e instanceof DatabaseException) { - $index->setAttribute('error', $e->getMessage()); - } - $dbForProject->updateDocument( - 'indexes', - $index->getId(), - $index->setAttribute('status', 'failed') - ); - } finally { - $this->trigger($database, $collection, $index, $project, $projectId, $events); - } - - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); - } - - /** - * @param Document $database - * @param Document $collection - * @param Document $index - * @param Document $project - * @param Database $dbForConsole - * @param Database $dbForProject - * @return void - * @throws Authorization - * @throws Conflict - * @throws Structure - * @throws DatabaseException - */ - private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void - { - if ($collection->isEmpty()) { - throw new Exception('Missing collection'); - } - if ($index->isEmpty()) { - throw new Exception('Missing index'); - } - - $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'indexId' => $index->getId() - ]); - $key = $index->getAttribute('key'); - $status = $index->getAttribute('status', ''); - $project = $dbForConsole->getDocument('projects', $projectId); - - try { - if ($status !== 'failed' && !$dbForProject->deleteIndex('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { - throw new DatabaseException('Failed to delete index'); - } - $dbForProject->deleteDocument('indexes', $index->getId()); - $index->setAttribute('status', 'deleted'); - } catch (\Exception $e) { - Console::error($e->getMessage()); - - if ($e instanceof DatabaseException) { - $index->setAttribute('error', $e->getMessage()); - } - $dbForProject->updateDocument( - 'indexes', - $index->getId(), - $index->setAttribute('status', 'stuck') - ); - } finally { - $this->trigger($database, $collection, $index, $project, $projectId, $events); - } - - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId()); - } - - /** - * @param Document $database - * @param Document $project - * @param $dbForProject - * @return void - * @throws Exception - */ - protected function deleteDatabase(Document $database, Document $project, $dbForProject): void - { - $this->deleteByGroup('database_' . $database->getInternalId(), [], $dbForProject, function ($collection) use ($database, $project, $dbForProject) { - $this->deleteCollection($database, $collection, $project, $dbForProject); - }); - - $dbForProject->deleteCollection('database_' . $database->getInternalId()); - - $this->deleteAuditLogsByResource('database/' . $database->getId(), $project, $dbForProject); - } - - /** - * @param Document $database - * @param Document $collection - * @param Document $project - * @param Database $dbForProject - * @return void - * @throws Authorization - * @throws Conflict - * @throws DatabaseException - * @throws Restricted - * @throws Structure - */ - protected function deleteCollection(Document $database, Document $collection, Document $project, Database $dbForProject): void - { - if ($collection->isEmpty()) { - throw new Exception('Missing collection'); - } - - $collectionId = $collection->getId(); - $collectionInternalId = $collection->getInternalId(); - $databaseId = $database->getId(); - $databaseInternalId = $database->getInternalId(); - - $relationships = \array_filter( - $collection->getAttribute('attributes'), - fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP - ); - - foreach ($relationships as $relationship) { - if (!$relationship['twoWay']) { - continue; - } - $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); - $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); - $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); - } - - $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); - - $this->deleteByGroup('attributes', [ - Query::equal('databaseInternalId', [$databaseInternalId]), - Query::equal('collectionInternalId', [$collectionInternalId]) - ], $dbForProject); - - $this->deleteByGroup('indexes', [ - Query::equal('databaseInternalId', [$databaseInternalId]), - Query::equal('collectionInternalId', [$collectionInternalId]) - ], $dbForProject); - - $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project, $dbForProject); - } - - /** - * @param string $resource - * @param Document $project - * @param Database $dbForProject - * @return void - * @throws Exception - */ - protected function deleteAuditLogsByResource(string $resource, Document $project, Database $dbForProject): void - { - $this->deleteByGroup(Audit::COLLECTION, [ - Query::equal('resource', [$resource]) - ], $dbForProject); - } - - /** - * @param string $collection collectionID - * @param array $queries - * @param Database $database - * @param callable|null $callback - * @return void - * @throws Exception - */ - protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void - { - $count = 0; - $chunk = 0; - $limit = 50; - $sum = $limit; - - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); - - $sum = count($results); - - Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); - - foreach ($results as $document) { - if ($database->deleteDocument($document->getCollection(), $document->getId())) { - Console::success('Deleted document "' . $document->getId() . '" successfully'); - - if (\is_callable($callback)) { - $callback($document); - } - } else { - Console::error('Failed to delete document: ' . $document->getId()); - } - $count++; - } - } - - $executionEnd = \microtime(true); - - Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); - } - - protected function trigger( - Document $database, - Document $collection, - Document $attribute, - Document $project, - string $projectId, - array $events - ): void { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project, - ); - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); - } -} diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php deleted file mode 100644 index 7a20212c9c..0000000000 --- a/src/Appwrite/Platform/Workers/Mails.php +++ /dev/null @@ -1,126 +0,0 @@ -desc('Mails worker') - ->inject('message') - ->inject('register') - ->callback(fn($message, $register) => $this->action($message, $register)); - } - - /** - * @param Message $message - * @param Registry $register - * @throws \PHPMailer\PHPMailer\Exception - * @return void - * @throws Exception - */ - public function action(Message $message, Registry $register): void - { - Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP); - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - $smtp = $payload['smtp']; - - if (empty($smtp) && empty(App::getEnv('_APP_SMTP_HOST'))) { - Console::info('Skipped mail processing. No SMTP configuration has been set.'); - return; - } - - $recipient = $payload['recipient']; - $subject = $payload['subject']; - $variables = $payload['variables']; - $name = $payload['name']; - $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); - - foreach ($variables as $key => $value) { - $body->setParam('{{' . $key . '}}', $value); - } - - $body = $body->render(); - - /** @var PHPMailer $mail */ - $mail = empty($smtp) - ? $register->get('smtp') - : $this->getMailer($smtp); - - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - $mail->addAddress($recipient, $name); - $mail->Subject = $subject; - $mail->Body = $body; - $mail->AltBody = \strip_tags($body); - - try { - $mail->send(); - } catch (\Exception $error) { - throw new Exception('Error sending mail: ' . $error->getMessage(), 500); - } - } - - /** - * @param array $smtp - * @return PHPMailer - * @throws \PHPMailer\PHPMailer\Exception - */ - protected function getMailer(array $smtp): PHPMailer - { - $mail = new PHPMailer(true); - - $mail->isSMTP(); - - $username = $smtp['username']; - $password = $smtp['password']; - - $mail->XMailer = 'Appwrite Mailer'; - $mail->Host = $smtp['host']; - $mail->Port = $smtp['port']; - $mail->SMTPAuth = (!empty($username) && !empty($password)); - $mail->Username = $username; - $mail->Password = $password; - $mail->SMTPSecure = $smtp['secure']; - $mail->SMTPAutoTLS = false; - $mail->CharSet = 'UTF-8'; - - $mail->setFrom($smtp['senderEmail'], $smtp['senderName']); - - if (!empty($smtp['replyTo'])) { - $mail->addReplyTo($smtp['replyTo'], $smtp['senderName']); - } - - $mail->isHTML(); - - return $mail; - } -} diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php deleted file mode 100644 index 31b0df59a3..0000000000 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ /dev/null @@ -1,330 +0,0 @@ -desc('Migrations worker') - ->inject('message') - ->inject('dbForProject') - ->inject('dbForConsole') - ->callback(fn(Message $message, Database $dbForProject, Database $dbForConsole) => $this->action($message, $dbForProject, $dbForConsole)); - } - - /** - * @param Message $message - * @param Database $dbForProject - * @param Database $dbForConsole - * @return void - * @throws Exception - */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole): void - { - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - $events = $payload['events'] ?? []; - $project = new Document($payload['project'] ?? []); - $migration = new Document($payload['migration'] ?? []); - - if ($project->getId() === 'console') { - return; - } - - $this->dbForProject = $dbForProject; - $this->dbForConsole = $dbForConsole; - - /** - * Handle Event execution. - */ - if (! empty($events)) { - return; - } - - $this->processMigration($project, $migration); - } - - /** - * @param string $source - * @param array $credentials - * @return Source - * @throws Exception - */ - protected function processSource(string $source, array $credentials): Source - { - return match ($source) { - Firebase::getName() => new Firebase( - json_decode($credentials['serviceAccount'], true), - ), - Supabase::getName() => new Supabase( - $credentials['endpoint'], - $credentials['apiKey'], - $credentials['databaseHost'], - 'postgres', - $credentials['username'], - $credentials['password'], - $credentials['port'], - ), - NHost::getName() => new NHost( - $credentials['subdomain'], - $credentials['region'], - $credentials['adminSecret'], - $credentials['database'], - $credentials['username'], - $credentials['password'], - $credentials['port'], - ), - Appwrite::getName() => new Appwrite($credentials['projectId'], str_starts_with($credentials['endpoint'], 'http://localhost/v1') ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey']), - default => throw new \Exception('Invalid source type'), - }; - } - - /** - * @throws Authorization - * @throws Structure - * @throws Conflict - * @throws \Utopia\Database\Exception - * @throws Exception - */ - protected function updateMigrationDocument(Document $migration, Document $project): Document - { - /** Trigger Realtime */ - $allEvents = Event::generateEvents('migrations.[migrationId].update', [ - 'migrationId' => $migration->getId(), - ]); - - $target = Realtime::fromPayload( - event: $allEvents[0], - payload: $migration, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $migration->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - ); - - Realtime::send( - projectId: $project->getId(), - payload: $migration->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - ); - - return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); - } - - /** - * @param Document $apiKey - * @return void - * @throws \Utopia\Database\Exception - * @throws Authorization - * @throws Conflict - * @throws Restricted - * @throws Structure - */ - protected function removeAPIKey(Document $apiKey): void - { - $this->dbForConsole->deleteDocument('keys', $apiKey->getId()); - } - - /** - * @param Document $project - * @return Document - * @throws Authorization - * @throws Structure - * @throws \Utopia\Database\Exception - * @throws Exception - */ - protected function generateAPIKey(Document $project): Document - { - $generatedSecret = bin2hex(\random_bytes(128)); - - $key = new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'projectInternalId' => $project->getInternalId(), - 'projectId' => $project->getId(), - 'name' => 'Transfer API Key', - 'scopes' => [ - 'users.read', - 'users.write', - 'teams.read', - 'teams.write', - 'databases.read', - 'databases.write', - 'collections.read', - 'collections.write', - 'documents.read', - 'documents.write', - 'buckets.read', - 'buckets.write', - 'files.read', - 'files.write', - 'functions.read', - 'functions.write', - ], - 'expire' => null, - 'sdks' => [], - 'accessedAt' => null, - 'secret' => $generatedSecret, - ]); - - $this->dbForConsole->createDocument('keys', $key); - $this->dbForConsole->deleteCachedDocument('projects', $project->getId()); - - return $key; - } - - /** - * @param Document $project - * @param Document $migration - * @return void - * @throws Authorization - * @throws Conflict - * @throws Restricted - * @throws Structure - * @throws \Utopia\Database\Exception - */ - protected function processMigration(Document $project, Document $migration): void - { - /** - * @var Document $migrationDocument - * @var Transfer $transfer - */ - $migrationDocument = null; - $transfer = null; - $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); - $tempAPIKey = $this->generateAPIKey($projectDocument); - - try { - $migrationDocument = $this->dbForProject->getDocument('migrations', $migration->getId()); - $migrationDocument->setAttribute('stage', 'processing'); - $migrationDocument->setAttribute('status', 'processing'); - $this->updateMigrationDocument($migrationDocument, $projectDocument); - - $source = $this->processSource($migrationDocument->getAttribute('source'), $migrationDocument->getAttribute('credentials')); - - $source->report(); - - $destination = new DestinationsAppwrite( - $projectDocument->getId(), - 'http://appwrite/v1', - $tempAPIKey['secret'], - ); - - $transfer = new Transfer( - $source, - $destination - ); - - /** Start Transfer */ - $migrationDocument->setAttribute('stage', 'migrating'); - $this->updateMigrationDocument($migrationDocument, $projectDocument); - $transfer->run($migrationDocument->getAttribute('resources'), function () use ($migrationDocument, $transfer, $projectDocument) { - $migrationDocument->setAttribute('resourceData', json_encode($transfer->getCache())); - $migrationDocument->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); - - $this->updateMigrationDocument($migrationDocument, $projectDocument); - }); - - $errors = $transfer->getReport(Resource::STATUS_ERROR); - - if (count($errors) > 0) { - $migrationDocument->setAttribute('status', 'failed'); - $migrationDocument->setAttribute('stage', 'finished'); - - $errorMessages = []; - foreach ($errors as $error) { - $errorMessages[] = "Failed to transfer resource '{$error['id']}:{$error['resource']}' with message '{$error['message']}'"; - } - - $migrationDocument->setAttribute('errors', $errorMessages); - $this->updateMigrationDocument($migrationDocument, $projectDocument); - - return; - } - - $migrationDocument->setAttribute('status', 'completed'); - $migrationDocument->setAttribute('stage', 'finished'); - } catch (\Throwable $th) { - Console::error($th->getMessage()); - - if ($migrationDocument) { - Console::error($th->getMessage()); - Console::error($th->getTraceAsString()); - $migrationDocument->setAttribute('status', 'failed'); - $migrationDocument->setAttribute('stage', 'finished'); - $migrationDocument->setAttribute('errors', [$th->getMessage()]); - - return; - } - - if ($transfer) { - $errors = $transfer->getReport(Resource::STATUS_ERROR); - - if (count($errors) > 0) { - $migrationDocument->setAttribute('status', 'failed'); - $migrationDocument->setAttribute('stage', 'finished'); - $migrationDocument->setAttribute('errors', $errors); - } - } - } finally { - if ($migrationDocument) { - $this->updateMigrationDocument($migrationDocument, $projectDocument); - } - if ($tempAPIKey) { - $this->removeAPIKey($tempAPIKey); - } - } - } -} diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php deleted file mode 100644 index dd7b92bf5e..0000000000 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ /dev/null @@ -1,118 +0,0 @@ -desc('Webhooks worker') - ->inject('message') - ->callback(fn($message) => $this->action($message)); - } - - /** - * @param Message $message - * @return void - * @throws Exception - */ - public function action(Message $message): void - { - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - $events = $payload['events']; - $webhookPayload = json_encode($payload['payload']); - $project = new Document($payload['project']); - $user = new Document($payload['user'] ?? []); - - foreach ($project->getAttribute('webhooks', []) as $webhook) { - if (array_intersect($webhook->getAttribute('events', []), $events)) { - $this->execute($events, $webhookPayload, $webhook, $user, $project); - } - } - - if (!empty($this->errors)) { - throw new Exception(\implode(" / \n\n", $this->errors)); - } - } - - /** - * @param array $events - * @param string $payload - * @param Document $webhook - * @param Document $user - * @param Document $project - * @return void - */ - private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project): void - { - - $url = \rawurldecode($webhook->getAttribute('url')); - $signatureKey = $webhook->getAttribute('signatureKey'); - $signature = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); - $httpUser = $webhook->getAttribute('httpUser'); - $httpPass = $webhook->getAttribute('httpPass'); - $ch = \curl_init($webhook->getAttribute('url')); - - \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - \curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); - \curl_setopt($ch, CURLOPT_HEADER, 0); - \curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); - \curl_setopt($ch, CURLOPT_USERAGENT, \sprintf( - APP_USERAGENT, - App::getEnv('_APP_VERSION', 'UNKNOWN'), - App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY) - )); - \curl_setopt( - $ch, - CURLOPT_HTTPHEADER, - [ - 'Content-Type: application/json', - 'Content-Length: ' . \strlen($payload), - 'X-' . APP_NAME . '-Webhook-Id: ' . $webhook->getId(), - 'X-' . APP_NAME . '-Webhook-Events: ' . implode(',', $events), - 'X-' . APP_NAME . '-Webhook-Name: ' . $webhook->getAttribute('name', ''), - 'X-' . APP_NAME . '-Webhook-User-Id: ' . $user->getId(), - 'X-' . APP_NAME . '-Webhook-Project-Id: ' . $project->getId(), - 'X-' . APP_NAME . '-Webhook-Signature: ' . $signature, - ] - ); - - if (!$webhook->getAttribute('security', true)) { - \curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); - \curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - } - - if (!empty($httpUser) && !empty($httpPass)) { - \curl_setopt($ch, CURLOPT_USERPWD, "$httpUser:$httpPass"); - \curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); - } - - if (false === \curl_exec($ch)) { - $this->errors[] = \curl_error($ch) . ' in events ' . implode(', ', $events) . ' for webhook ' . $webhook->getAttribute('name'); - } - - \curl_close($ch); - } -} From 025a0292f4b61850c67a97e12305f9d3dfedf3a6 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 21 Oct 2023 07:02:44 -0400 Subject: [PATCH 003/406] Revert "Sync with main" This reverts commit 0ea1418132b194c173e6efe211042e985c73b075. --- .../examples/health/get-queue-builds.md | 18 + .../examples/health/get-queue-databases.md | 18 + .../examples/health/get-queue-deletes.md | 18 + .../examples/health/get-queue-mails.md | 18 + .../Platform/Workers/Certificates.php | 534 +++++++++++++++ src/Appwrite/Platform/Workers/Databases.php | 622 ++++++++++++++++++ src/Appwrite/Platform/Workers/Mails.php | 126 ++++ src/Appwrite/Platform/Workers/Migrations.php | 330 ++++++++++ src/Appwrite/Platform/Workers/Webhooks.php | 118 ++++ 9 files changed, 1802 insertions(+) create mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md create mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md create mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md create mode 100644 docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md create mode 100644 src/Appwrite/Platform/Workers/Certificates.php create mode 100644 src/Appwrite/Platform/Workers/Databases.php create mode 100644 src/Appwrite/Platform/Workers/Mails.php create mode 100644 src/Appwrite/Platform/Workers/Migrations.php create mode 100644 src/Appwrite/Platform/Workers/Webhooks.php diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md new file mode 100644 index 0000000000..a312fa7df0 --- /dev/null +++ b/docs/examples/1.4.x/console-web/examples/health/get-queue-builds.md @@ -0,0 +1,18 @@ +import { Client, Health } from "@appwrite.io/console"; + +const client = new Client(); + +const health = new Health(client); + +client + .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint + .setProject('5df5acd0d48c2') // Your project ID +; + +const promise = health.getQueueBuilds(); + +promise.then(function (response) { + console.log(response); // Success +}, function (error) { + console.log(error); // Failure +}); \ No newline at end of file diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md new file mode 100644 index 0000000000..481480a80d --- /dev/null +++ b/docs/examples/1.4.x/console-web/examples/health/get-queue-databases.md @@ -0,0 +1,18 @@ +import { Client, Health } from "@appwrite.io/console"; + +const client = new Client(); + +const health = new Health(client); + +client + .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint + .setProject('5df5acd0d48c2') // Your project ID +; + +const promise = health.getQueueDatabases(); + +promise.then(function (response) { + console.log(response); // Success +}, function (error) { + console.log(error); // Failure +}); \ No newline at end of file diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md new file mode 100644 index 0000000000..c1bbb9f655 --- /dev/null +++ b/docs/examples/1.4.x/console-web/examples/health/get-queue-deletes.md @@ -0,0 +1,18 @@ +import { Client, Health } from "@appwrite.io/console"; + +const client = new Client(); + +const health = new Health(client); + +client + .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint + .setProject('5df5acd0d48c2') // Your project ID +; + +const promise = health.getQueueDeletes(); + +promise.then(function (response) { + console.log(response); // Success +}, function (error) { + console.log(error); // Failure +}); \ No newline at end of file diff --git a/docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md b/docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md new file mode 100644 index 0000000000..8d6d68228a --- /dev/null +++ b/docs/examples/1.4.x/console-web/examples/health/get-queue-mails.md @@ -0,0 +1,18 @@ +import { Client, Health } from "@appwrite.io/console"; + +const client = new Client(); + +const health = new Health(client); + +client + .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint + .setProject('5df5acd0d48c2') // Your project ID +; + +const promise = health.getQueueMails(); + +promise.then(function (response) { + console.log(response); // Success +}, function (error) { + console.log(error); // Failure +}); \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php new file mode 100644 index 0000000000..02c1835dd5 --- /dev/null +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -0,0 +1,534 @@ +desc('Certificates worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('queueForMails') + ->inject('queueForEvents') + ->inject('queueForFunctions') + ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions)); + } + + /** + * @param Message $message + * @param Database $dbForConsole + * @param Mail $queueForMails + * @param Event $queueForEvents + * @param Func $queueForFunctions + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $document = new Document($payload['domain'] ?? []); + $domain = new Domain($document->getAttribute('domain', '')); + $skipRenewCheck = $payload['skipRenewCheck'] ?? false; + + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck); + } + + /** + * @param Domain $domain + * @param Database $dbForConsole + * @param Mail $queueForMails + * @param Event $queueForEvents + * @param Func $queueForFunctions + * @param bool $skipRenewCheck + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void + { + /** + * 1. Read arguments and validate domain + * 2. Get main domain + * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) + * 4. Validate security email. Cannot be empty, required by LetsEncrypt + * 5. Validate renew date with certificate file, unless requested to skip by parameter + * 6. Issue a certificate using certbot CLI + * 7. Update 'log' attribute on certificate document with Certbot message + * 8. Create storage folder for certificate, if not ready already + * 9. Move certificates from Certbot location to our Storage + * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 11. Read certificate file and update 'renewDate' on certificate document + * 12. Update 'issueDate' and 'attempts' on certificate + * + * If at any point unexpected error occurs, program stops without applying changes to document, and error is thrown into worker + * + * If code stops with expected error: + * 1. 'log' attribute on document is updated with error message + * 2. 'attempts' amount is increased + * 3. Console log is shown + * 4. Email is sent to security email + * + * Unless unexpected error occurs, at the end, we: + * 1. Update 'updated' attribute on document + * 2. Save document to database + * 3. Update all domains documents with current certificate ID + * + * Note: Renewals are checked and scheduled from maintenence worker + */ + + // Get current certificate + $certificate = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain->get()])]); + + // If we don't have certificate for domain yet, let's create new document. At the end we save it + if (!$certificate) { + $certificate = new Document(); + $certificate->setAttribute('domain', $domain->get()); + } + + $success = false; + + try { + // Email for alerts is required by LetsEncrypt + $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_SYSTEM_SECURITY_EMAIL_ADDRESS) to issue an SSL certificate.'); + } + + // Validate domain and DNS records. Skip if job is forced + if (!$skipRenewCheck) { + $mainDomain = $this->getMainDomain(); + $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; + $this->validateDomain($domain, $isMainDomain); + } + + // If certificate exists already, double-check expiry date. Skip if job is forced + if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { + throw new Exception('Renew isn\'t required.'); + } + + // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate + $folder = ID::unique(); + + // Generate certificate files using Let's Encrypt + $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); + + // Command succeeded, store all data into document + $logs = 'Certificate successfully generated.'; + $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB + + + // Give certificates to Traefik + $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); + + // Update certificate info stored in database + $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('attempts', 0); + $certificate->setAttribute('issueDate', DateTime::now()); + $success = true; + } catch (Throwable $e) { + $logs = $e->getMessage(); + + // Set exception as log in certificate document + $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB + + // Increase attempts count + $attempts = $certificate->getAttribute('attempts', 0) + 1; + $certificate->setAttribute('attempts', $attempts); + + // Store cuttent time as renew date to ensure another attempt in next maintenance cycle + $certificate->setAttribute('renewDate', DateTime::now()); + + // Send email to security email + $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails); + } finally { + // All actions result in new updatedAt date + $certificate->setAttribute('updated', DateTime::now()); + + // Save all changes we made to certificate document into database + $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForConsole, $queueForEvents, $queueForFunctions); + } + } + + /** + * Save certificate data into database. + * + * @param string $domain Domain name that certificate is for + * @param Document $certificate Certificate document that we need to save + * @param bool $success + * @param Database $dbForConsole Database connection for console + * @param Event $queueForEvents + * @param Func $queueForFunctions + * @return void + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Conflict + * @throws Structure + */ + private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void + { + // Check if update or insert required + $certificateDocument = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]); + if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { + // Merge new data with current data + $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); + $certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); + } else { + $certificate->removeAttribute('$internalId'); + $certificate = $dbForConsole->createDocument('certificates', $certificate); + } + + $certificateId = $certificate->getId(); + $this->updateDomainDocuments($certificateId, $domain, $success, $dbForConsole, $queueForEvents, $queueForFunctions); + } + + /** + * Get main domain. Needed as we do different checks for main and non-main domains. + * + * @return null|string Returns main domain. If null, there is no main domain yet. + */ + private function getMainDomain(): ?string + { + $envDomain = App::getEnv('_APP_DOMAIN', ''); + if (!empty($envDomain) && $envDomain !== 'localhost') { + return $envDomain; + } + + return null; + } + + /** + * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * - Domain must have proper DNS record + * + * @param Domain $domain Domain which we validate + * @param bool $isMainDomain In case of master domain, we look for different DNS configurations + * + * @return void + * @throws Exception + */ + private function validateDomain(Domain $domain, bool $isMainDomain): void + { + if (empty($domain->get())) { + throw new Exception('Missing certificate domain.'); + } + + if (!$domain->isKnown() || $domain->isTest()) { + throw new Exception('Unknown public suffix for domain.'); + } + + if (!$isMainDomain) { + // TODO: Would be awesome to also support A/AAAA records here. Maybe dry run? + // Validate if domain target is properly configured + $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); + + if (!$target->isKnown() || $target->isTest()) { + throw new Exception('Unreachable CNAME target (' . $target->get() . '), please use a domain with a public suffix.'); + } + + // Verify domain with DNS records + $validator = new CNAME($target->get()); + if (!$validator->isValid($domain->get())) { + throw new Exception('Failed to verify domain DNS records.'); + } + } else { + // Main domain validation + // TODO: Would be awesome to check A/AAAA record here. Maybe dry run? + } + } + + /** + * Reads expiry date of certificate from file and decides if renewal is required or not. + * + * @param string $domain Domain for which we check certificate file + * @return bool True, if certificate needs to be renewed + * @throws Exception + */ + private function isRenewRequired(string $domain): bool + { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $validTo = null; + + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60 * 60 * 24 * 30); + if ($validTo - $expiryInAdvance > \time()) { + return false; + } + } + + return true; + } + + /** + * LetsEncrypt communication to issue certificate (using certbot CLI) + * + * @param string $folder Folder into which certificates should be generated + * @param string $domain Domain to generate certificate for + * @return array Named array with keys 'stdout' and 'stderr', both string + * @throws Exception + */ + private function issueCertificate(string $folder, string $domain, string $email): array + { + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $email + . " --cert-name " . $folder + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", '', $stdout, $stderr); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + return [ + 'stdout' => $stdout, + 'stderr' => $stderr + ]; + } + + /** + * Read new renew date from certificate file generated by Let's Encrypt + * + * @param string $domain Domain which certificate was generated for + * @return string + * @throws \Utopia\Database\Exception + */ + private function getRenewDate(string $domain): string + { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? null; + $dt = (new \DateTime())->setTimestamp($validTo); + return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days + } + + /** + * Method to take files from Let's Encrypt, and put it into Traefik. + * + * @param string $domain Domain which certificate was generated for + * @param string $folder Folder in which certificates were generated + * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error + * @return void + * @throws Exception + */ + private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void + { + + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); + } + + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); + } + + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); + } + + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); + } + + $config = \implode(PHP_EOL, [ + "tls:", + " certificates:", + " - certFile: /storage/certificates/{$domain}/fullchain.pem", + " keyFile: /storage/certificates/{$domain}/privkey.pem" + ]); + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + } + + /** + * Method to make sure information about error is delivered to admnistrator. + * + * @param string $domain Domain that caused the error + * @param string $errorMessage Verbose error message + * @param int $attempt How many times it failed already + * @param Mail $queueForMails + * @return void + * @throws Exception + */ + private function notifyError(string $domain, string $errorMessage, int $attempt, Mail $queueForMails): void + { + // Log error into console + Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage); + + // Send mail to administratore mail + + $locale = new Locale(App::getEnv('_APP_LOCALE', 'en')); + if (!$locale->getText('emails.sender') || !$locale->getText("emails.certificate.hello") || !$locale->getText("emails.certificate.subject") || !$locale->getText("emails.certificate.body") || !$locale->getText("emails.certificate.footer") || !$locale->getText("emails.certificate.thanks") || !$locale->getText("emails.certificate.signature")) { + $locale->setDefault('en'); + } + + $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); + + $subject = \sprintf($locale->getText("emails.certificate.subject"), $domain); + $body + ->setParam('{{domain}}', $domain) + ->setParam('{{error}}', $errorMessage) + ->setParam('{{attempt}}', $attempt) + ->setParam('{{subject}}', $subject) + ->setParam('{{hello}}', $locale->getText("emails.certificate.hello")) + ->setParam('{{body}}', $locale->getText("emails.certificate.body")) + ->setParam('{{redirect}}', 'https://' . $domain) + ->setParam('{{footer}}', $locale->getText("emails.certificate.footer")) + ->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks")) + ->setParam('{{signature}}', $locale->getText("emails.certificate.signature")) + ->setParam('{{project}}', 'Console') + ->setParam('{{direction}}', $locale->getText('settings.direction')) + ->setParam('{{bg-body}}', '#f7f7f7') + ->setParam('{{bg-content}}', '#ffffff') + ->setParam('{{text-content}}', '#000000'); + + $queueForMails + ->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')) + ->setBody($body->render()) + ->setName('Appwrite Administrator') + ->trigger(); + } + + /** + * Update all existing domain documents so they have relation to correct certificate document. + * This solved issues: + * - when adding a domain for which there is already a certificate + * - when renew creates new document? It might? + * - overall makes it more reliable + * + * @param string $certificateId ID of a new or updated certificate document + * @param string $domain Domain that is affected by new certificate + * @param bool $success Was certificate generation successful? + * + * @return void + */ + private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void + { + + $rule = $dbForConsole->findOne('rules', [ + Query::equal('domain', [$domain]), + ]); + + if ($rule !== false && !$rule->isEmpty()) { + $rule->setAttribute('certificateId', $certificateId); + $rule->setAttribute('status', $success ? 'verified' : 'unverified'); + $dbForConsole->updateDocument('rules', $rule->getId(), $rule); + + $projectId = $rule->getAttribute('projectId'); + + // Skip events for console project (triggered by auto-ssl generation for 1 click setups) + if ($projectId === 'console') { + return; + } + + $project = $dbForConsole->getDocument('projects', $projectId); + + /** Trigger Webhook */ + $ruleModel = new Rule(); + $queueForEvents + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) + ->trigger(); + + + /** Trigger Functions */ + $queueForFunctions + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) + ->trigger(); + + /** Trigger realtime event */ + $allEvents = Event::generateEvents('rules.[ruleId].update', [ + 'ruleId' => $rule->getId(), + ]); + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $rule, + project: $project + ); + Realtime::send( + projectId: 'console', + payload: $rule->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + Realtime::send( + projectId: $project->getId(), + payload: $rule->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + } + } +} diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php new file mode 100644 index 0000000000..e0ec75e1d4 --- /dev/null +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -0,0 +1,622 @@ +desc('Databases worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('dbForProject') + ->callback(fn($message, $dbForConsole, $dbForProject) => $this->action($message, $dbForConsole, $dbForProject)); + } + + /** + * @param Message $message + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws \Exception + */ + public function action(Message $message, Database $dbForConsole, Database $dbForProject): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + $type = $payload['type']; + $project = new Document($payload['project']); + $collection = new Document($payload['collection'] ?? []); + $document = new Document($payload['document'] ?? []); + $database = new Document($payload['database'] ?? []); + + if ($database->isEmpty()) { + throw new Exception('Missing database'); + } + + match (strval($type)) { + DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), + DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), + DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), + DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), + DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), + DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), + default => Console::error('No database operation for type: ' . $type), + }; + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $attribute + * @param Document $project + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws \Exception + */ + private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } + + $projectId = $project->getId(); + + $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId(), + 'attributeId' => $attribute->getId() + ]); + /** + * TODO @christyjacob4 verify if this is still the case + * Fetch attribute from the database, since with Resque float values are loosing informations. + */ + $attribute = $dbForProject->getDocument('attributes', $attribute->getId()); + + $collectionId = $collection->getId(); + $key = $attribute->getAttribute('key', ''); + $type = $attribute->getAttribute('type', ''); + $size = $attribute->getAttribute('size', 0); + $required = $attribute->getAttribute('required', false); + $default = $attribute->getAttribute('default', null); + $signed = $attribute->getAttribute('signed', true); + $array = $attribute->getAttribute('array', false); + $format = $attribute->getAttribute('format', ''); + $formatOptions = $attribute->getAttribute('formatOptions', []); + $filters = $attribute->getAttribute('filters', []); + $options = $attribute->getAttribute('options', []); + $project = $dbForConsole->getDocument('projects', $projectId); + + + try { + switch ($type) { + case Database::VAR_RELATIONSHIP: + $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']); + if ($relatedCollection->isEmpty()) { + throw new DatabaseException('Collection not found'); + } + + if ( + !$dbForProject->createRelationship( + collection: 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + relatedCollection: 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), + type: $options['relationType'], + twoWay: $options['twoWay'], + id: $key, + twoWayKey: $options['twoWayKey'], + onDelete: $options['onDelete'], + ) + ) { + throw new DatabaseException('Failed to create Attribute'); + } + + if ($options['twoWay']) { + $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']); + $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'available')); + } + break; + default: + if (!$dbForProject->createAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) { + throw new \Exception('Failed to create Attribute'); + } + } + + $dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available')); + } catch (\Exception $e) { + Console::error($e->getMessage()); + + if ($e instanceof DatabaseException) { + $attribute->setAttribute('error', $e->getMessage()); + if (isset($relatedAttribute)) { + $relatedAttribute->setAttribute('error', $e->getMessage()); + } + } + + $dbForProject->updateDocument( + 'attributes', + $attribute->getId(), + $attribute->setAttribute('status', 'failed') + ); + + if (isset($relatedAttribute)) { + $dbForProject->updateDocument( + 'attributes', + $relatedAttribute->getId(), + $relatedAttribute->setAttribute('status', 'failed') + ); + } + } finally { + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); + } + + if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) { + $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); + } + + $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $attribute + * @param Document $project + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws \Exception + **/ + private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } + + $projectId = $project->getId(); + + $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId(), + 'attributeId' => $attribute->getId() + ]); + $collectionId = $collection->getId(); + $key = $attribute->getAttribute('key', ''); + $status = $attribute->getAttribute('status', ''); + $type = $attribute->getAttribute('type', ''); + $project = $dbForConsole->getDocument('projects', $projectId); + $options = $attribute->getAttribute('options', []); + $relatedAttribute = new Document(); + $relatedCollection = new Document(); + // possible states at this point: + // - available: should not land in queue; controller flips these to 'deleting' + // - processing: hasn't finished creating + // - deleting: was available, in deletion queue for first time + // - failed: attribute was never created + // - stuck: attribute was available but cannot be removed + + try { + if ($status !== 'failed') { + if ($type === Database::VAR_RELATIONSHIP) { + if ($options['twoWay']) { + $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']); + if ($relatedCollection->isEmpty()) { + throw new DatabaseException('Collection not found'); + } + $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']); + } + + if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { + $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck')); + throw new DatabaseException('Failed to delete Relationship'); + } + } elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { + throw new DatabaseException('Failed to delete Attribute'); + } + } + + $dbForProject->deleteDocument('attributes', $attribute->getId()); + + if (!$relatedAttribute->isEmpty()) { + $dbForProject->deleteDocument('attributes', $relatedAttribute->getId()); + } + } catch (\Exception $e) { + Console::error($e->getMessage()); + + if ($e instanceof DatabaseException) { + $attribute->setAttribute('error', $e->getMessage()); + if (!$relatedAttribute->isEmpty()) { + $relatedAttribute->setAttribute('error', $e->getMessage()); + } + } + $dbForProject->updateDocument( + 'attributes', + $attribute->getId(), + $attribute->setAttribute('status', 'stuck') + ); + if (!$relatedAttribute->isEmpty()) { + $dbForProject->updateDocument( + 'attributes', + $relatedAttribute->getId(), + $relatedAttribute->setAttribute('status', 'stuck') + ); + } + } finally { + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); + } + + // The underlying database removes/rebuilds indexes when attribute is removed + // Update indexes table with changes + /** @var Document[] $indexes */ + $indexes = $collection->getAttribute('indexes', []); + + foreach ($indexes as $index) { + /** @var string[] $attributes */ + $attributes = $index->getAttribute('attributes'); + $lengths = $index->getAttribute('lengths'); + $orders = $index->getAttribute('orders'); + + $found = \array_search($key, $attributes); + + if ($found !== false) { + // If found, remove entry from attributes, lengths, and orders + // array_values wraps array_diff to reindex array keys + // when found attribute is removed from array + $attributes = \array_values(\array_diff($attributes, [$attributes[$found]])); + $lengths = \array_values(\array_diff($lengths, isset($lengths[$found]) ? [$lengths[$found]] : [])); + $orders = \array_values(\array_diff($orders, isset($orders[$found]) ? [$orders[$found]] : [])); + + if (empty($attributes)) { + $dbForProject->deleteDocument('indexes', $index->getId()); + } else { + $index + ->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN) + ->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN) + ->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN); + + // Check if an index exists with the same attributes and orders + $exists = false; + foreach ($indexes as $existing) { + if ( + $existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself + && $existing->getAttribute('attributes') === $index->getAttribute('attributes') + && $existing->getAttribute('orders') === $index->getAttribute('orders') + ) { + $exists = true; + break; + } + } + + if ($exists) { // Delete the duplicate if created, else update in db + $this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject); + } else { + $dbForProject->updateDocument('indexes', $index->getId(), $index); + } + } + } + } + + $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); + $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + + if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) { + $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); + $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + } + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $index + * @param Document $project + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws Structure + * @throws DatabaseException + */ + private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + + $projectId = $project->getId(); + + $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId(), + 'indexId' => $index->getId() + ]); + $collectionId = $collection->getId(); + $key = $index->getAttribute('key', ''); + $type = $index->getAttribute('type', ''); + $attributes = $index->getAttribute('attributes', []); + $lengths = $index->getAttribute('lengths', []); + $orders = $index->getAttribute('orders', []); + $project = $dbForConsole->getDocument('projects', $projectId); + + try { + if (!$dbForProject->createIndex('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $attributes, $lengths, $orders)) { + throw new DatabaseException('Failed to create Index'); + } + $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available')); + } catch (\Exception $e) { + Console::error($e->getMessage()); + + if ($e instanceof DatabaseException) { + $index->setAttribute('error', $e->getMessage()); + } + $dbForProject->updateDocument( + 'indexes', + $index->getId(), + $index->setAttribute('status', 'failed') + ); + } finally { + $this->trigger($database, $collection, $index, $project, $projectId, $events); + } + + $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $index + * @param Document $project + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws Structure + * @throws DatabaseException + */ + private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + + $projectId = $project->getId(); + + $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId(), + 'indexId' => $index->getId() + ]); + $key = $index->getAttribute('key'); + $status = $index->getAttribute('status', ''); + $project = $dbForConsole->getDocument('projects', $projectId); + + try { + if ($status !== 'failed' && !$dbForProject->deleteIndex('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { + throw new DatabaseException('Failed to delete index'); + } + $dbForProject->deleteDocument('indexes', $index->getId()); + $index->setAttribute('status', 'deleted'); + } catch (\Exception $e) { + Console::error($e->getMessage()); + + if ($e instanceof DatabaseException) { + $index->setAttribute('error', $e->getMessage()); + } + $dbForProject->updateDocument( + 'indexes', + $index->getId(), + $index->setAttribute('status', 'stuck') + ); + } finally { + $this->trigger($database, $collection, $index, $project, $projectId, $events); + } + + $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId()); + } + + /** + * @param Document $database + * @param Document $project + * @param $dbForProject + * @return void + * @throws Exception + */ + protected function deleteDatabase(Document $database, Document $project, $dbForProject): void + { + $this->deleteByGroup('database_' . $database->getInternalId(), [], $dbForProject, function ($collection) use ($database, $project, $dbForProject) { + $this->deleteCollection($database, $collection, $project, $dbForProject); + }); + + $dbForProject->deleteCollection('database_' . $database->getInternalId()); + + $this->deleteAuditLogsByResource('database/' . $database->getId(), $project, $dbForProject); + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws DatabaseException + * @throws Restricted + * @throws Structure + */ + protected function deleteCollection(Document $database, Document $collection, Document $project, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + + $collectionId = $collection->getId(); + $collectionInternalId = $collection->getInternalId(); + $databaseId = $database->getId(); + $databaseInternalId = $database->getInternalId(); + + $relationships = \array_filter( + $collection->getAttribute('attributes'), + fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP + ); + + foreach ($relationships as $relationship) { + if (!$relationship['twoWay']) { + continue; + } + $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); + $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); + $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); + $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); + } + + $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); + + $this->deleteByGroup('attributes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteByGroup('indexes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project, $dbForProject); + } + + /** + * @param string $resource + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Exception + */ + protected function deleteAuditLogsByResource(string $resource, Document $project, Database $dbForProject): void + { + $this->deleteByGroup(Audit::COLLECTION, [ + Query::equal('resource', [$resource]) + ], $dbForProject); + } + + /** + * @param string $collection collectionID + * @param array $queries + * @param Database $database + * @param callable|null $callback + * @return void + * @throws Exception + */ + protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + { + $count = 0; + $chunk = 0; + $limit = 50; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + if ($database->deleteDocument($document->getCollection(), $document->getId())) { + Console::success('Deleted document "' . $document->getId() . '" successfully'); + + if (\is_callable($callback)) { + $callback($document); + } + } else { + Console::error('Failed to delete document: ' . $document->getId()); + } + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + + protected function trigger( + Document $database, + Document $collection, + Document $attribute, + Document $project, + string $projectId, + array $events + ): void { + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $events[0], + payload: $attribute, + project: $project, + ); + Realtime::send( + projectId: 'console', + payload: $attribute->getArrayCopy(), + events: $events, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'projectId' => $projectId, + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId() + ] + ); + } +} diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php new file mode 100644 index 0000000000..7a20212c9c --- /dev/null +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -0,0 +1,126 @@ +desc('Mails worker') + ->inject('message') + ->inject('register') + ->callback(fn($message, $register) => $this->action($message, $register)); + } + + /** + * @param Message $message + * @param Registry $register + * @throws \PHPMailer\PHPMailer\Exception + * @return void + * @throws Exception + */ + public function action(Message $message, Registry $register): void + { + Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP); + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $smtp = $payload['smtp']; + + if (empty($smtp) && empty(App::getEnv('_APP_SMTP_HOST'))) { + Console::info('Skipped mail processing. No SMTP configuration has been set.'); + return; + } + + $recipient = $payload['recipient']; + $subject = $payload['subject']; + $variables = $payload['variables']; + $name = $payload['name']; + $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); + + foreach ($variables as $key => $value) { + $body->setParam('{{' . $key . '}}', $value); + } + + $body = $body->render(); + + /** @var PHPMailer $mail */ + $mail = empty($smtp) + ? $register->get('smtp') + : $this->getMailer($smtp); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + $mail->addAddress($recipient, $name); + $mail->Subject = $subject; + $mail->Body = $body; + $mail->AltBody = \strip_tags($body); + + try { + $mail->send(); + } catch (\Exception $error) { + throw new Exception('Error sending mail: ' . $error->getMessage(), 500); + } + } + + /** + * @param array $smtp + * @return PHPMailer + * @throws \PHPMailer\PHPMailer\Exception + */ + protected function getMailer(array $smtp): PHPMailer + { + $mail = new PHPMailer(true); + + $mail->isSMTP(); + + $username = $smtp['username']; + $password = $smtp['password']; + + $mail->XMailer = 'Appwrite Mailer'; + $mail->Host = $smtp['host']; + $mail->Port = $smtp['port']; + $mail->SMTPAuth = (!empty($username) && !empty($password)); + $mail->Username = $username; + $mail->Password = $password; + $mail->SMTPSecure = $smtp['secure']; + $mail->SMTPAutoTLS = false; + $mail->CharSet = 'UTF-8'; + + $mail->setFrom($smtp['senderEmail'], $smtp['senderName']); + + if (!empty($smtp['replyTo'])) { + $mail->addReplyTo($smtp['replyTo'], $smtp['senderName']); + } + + $mail->isHTML(); + + return $mail; + } +} diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php new file mode 100644 index 0000000000..31b0df59a3 --- /dev/null +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -0,0 +1,330 @@ +desc('Migrations worker') + ->inject('message') + ->inject('dbForProject') + ->inject('dbForConsole') + ->callback(fn(Message $message, Database $dbForProject, Database $dbForConsole) => $this->action($message, $dbForProject, $dbForConsole)); + } + + /** + * @param Message $message + * @param Database $dbForProject + * @param Database $dbForConsole + * @return void + * @throws Exception + */ + public function action(Message $message, Database $dbForProject, Database $dbForConsole): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $events = $payload['events'] ?? []; + $project = new Document($payload['project'] ?? []); + $migration = new Document($payload['migration'] ?? []); + + if ($project->getId() === 'console') { + return; + } + + $this->dbForProject = $dbForProject; + $this->dbForConsole = $dbForConsole; + + /** + * Handle Event execution. + */ + if (! empty($events)) { + return; + } + + $this->processMigration($project, $migration); + } + + /** + * @param string $source + * @param array $credentials + * @return Source + * @throws Exception + */ + protected function processSource(string $source, array $credentials): Source + { + return match ($source) { + Firebase::getName() => new Firebase( + json_decode($credentials['serviceAccount'], true), + ), + Supabase::getName() => new Supabase( + $credentials['endpoint'], + $credentials['apiKey'], + $credentials['databaseHost'], + 'postgres', + $credentials['username'], + $credentials['password'], + $credentials['port'], + ), + NHost::getName() => new NHost( + $credentials['subdomain'], + $credentials['region'], + $credentials['adminSecret'], + $credentials['database'], + $credentials['username'], + $credentials['password'], + $credentials['port'], + ), + Appwrite::getName() => new Appwrite($credentials['projectId'], str_starts_with($credentials['endpoint'], 'http://localhost/v1') ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey']), + default => throw new \Exception('Invalid source type'), + }; + } + + /** + * @throws Authorization + * @throws Structure + * @throws Conflict + * @throws \Utopia\Database\Exception + * @throws Exception + */ + protected function updateMigrationDocument(Document $migration, Document $project): Document + { + /** Trigger Realtime */ + $allEvents = Event::generateEvents('migrations.[migrationId].update', [ + 'migrationId' => $migration->getId(), + ]); + + $target = Realtime::fromPayload( + event: $allEvents[0], + payload: $migration, + project: $project + ); + + Realtime::send( + projectId: 'console', + payload: $migration->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + ); + + Realtime::send( + projectId: $project->getId(), + payload: $migration->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + ); + + return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); + } + + /** + * @param Document $apiKey + * @return void + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Conflict + * @throws Restricted + * @throws Structure + */ + protected function removeAPIKey(Document $apiKey): void + { + $this->dbForConsole->deleteDocument('keys', $apiKey->getId()); + } + + /** + * @param Document $project + * @return Document + * @throws Authorization + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Exception + */ + protected function generateAPIKey(Document $project): Document + { + $generatedSecret = bin2hex(\random_bytes(128)); + + $key = new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'projectInternalId' => $project->getInternalId(), + 'projectId' => $project->getId(), + 'name' => 'Transfer API Key', + 'scopes' => [ + 'users.read', + 'users.write', + 'teams.read', + 'teams.write', + 'databases.read', + 'databases.write', + 'collections.read', + 'collections.write', + 'documents.read', + 'documents.write', + 'buckets.read', + 'buckets.write', + 'files.read', + 'files.write', + 'functions.read', + 'functions.write', + ], + 'expire' => null, + 'sdks' => [], + 'accessedAt' => null, + 'secret' => $generatedSecret, + ]); + + $this->dbForConsole->createDocument('keys', $key); + $this->dbForConsole->deleteCachedDocument('projects', $project->getId()); + + return $key; + } + + /** + * @param Document $project + * @param Document $migration + * @return void + * @throws Authorization + * @throws Conflict + * @throws Restricted + * @throws Structure + * @throws \Utopia\Database\Exception + */ + protected function processMigration(Document $project, Document $migration): void + { + /** + * @var Document $migrationDocument + * @var Transfer $transfer + */ + $migrationDocument = null; + $transfer = null; + $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); + $tempAPIKey = $this->generateAPIKey($projectDocument); + + try { + $migrationDocument = $this->dbForProject->getDocument('migrations', $migration->getId()); + $migrationDocument->setAttribute('stage', 'processing'); + $migrationDocument->setAttribute('status', 'processing'); + $this->updateMigrationDocument($migrationDocument, $projectDocument); + + $source = $this->processSource($migrationDocument->getAttribute('source'), $migrationDocument->getAttribute('credentials')); + + $source->report(); + + $destination = new DestinationsAppwrite( + $projectDocument->getId(), + 'http://appwrite/v1', + $tempAPIKey['secret'], + ); + + $transfer = new Transfer( + $source, + $destination + ); + + /** Start Transfer */ + $migrationDocument->setAttribute('stage', 'migrating'); + $this->updateMigrationDocument($migrationDocument, $projectDocument); + $transfer->run($migrationDocument->getAttribute('resources'), function () use ($migrationDocument, $transfer, $projectDocument) { + $migrationDocument->setAttribute('resourceData', json_encode($transfer->getCache())); + $migrationDocument->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); + + $this->updateMigrationDocument($migrationDocument, $projectDocument); + }); + + $errors = $transfer->getReport(Resource::STATUS_ERROR); + + if (count($errors) > 0) { + $migrationDocument->setAttribute('status', 'failed'); + $migrationDocument->setAttribute('stage', 'finished'); + + $errorMessages = []; + foreach ($errors as $error) { + $errorMessages[] = "Failed to transfer resource '{$error['id']}:{$error['resource']}' with message '{$error['message']}'"; + } + + $migrationDocument->setAttribute('errors', $errorMessages); + $this->updateMigrationDocument($migrationDocument, $projectDocument); + + return; + } + + $migrationDocument->setAttribute('status', 'completed'); + $migrationDocument->setAttribute('stage', 'finished'); + } catch (\Throwable $th) { + Console::error($th->getMessage()); + + if ($migrationDocument) { + Console::error($th->getMessage()); + Console::error($th->getTraceAsString()); + $migrationDocument->setAttribute('status', 'failed'); + $migrationDocument->setAttribute('stage', 'finished'); + $migrationDocument->setAttribute('errors', [$th->getMessage()]); + + return; + } + + if ($transfer) { + $errors = $transfer->getReport(Resource::STATUS_ERROR); + + if (count($errors) > 0) { + $migrationDocument->setAttribute('status', 'failed'); + $migrationDocument->setAttribute('stage', 'finished'); + $migrationDocument->setAttribute('errors', $errors); + } + } + } finally { + if ($migrationDocument) { + $this->updateMigrationDocument($migrationDocument, $projectDocument); + } + if ($tempAPIKey) { + $this->removeAPIKey($tempAPIKey); + } + } + } +} diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php new file mode 100644 index 0000000000..dd7b92bf5e --- /dev/null +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -0,0 +1,118 @@ +desc('Webhooks worker') + ->inject('message') + ->callback(fn($message) => $this->action($message)); + } + + /** + * @param Message $message + * @return void + * @throws Exception + */ + public function action(Message $message): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $events = $payload['events']; + $webhookPayload = json_encode($payload['payload']); + $project = new Document($payload['project']); + $user = new Document($payload['user'] ?? []); + + foreach ($project->getAttribute('webhooks', []) as $webhook) { + if (array_intersect($webhook->getAttribute('events', []), $events)) { + $this->execute($events, $webhookPayload, $webhook, $user, $project); + } + } + + if (!empty($this->errors)) { + throw new Exception(\implode(" / \n\n", $this->errors)); + } + } + + /** + * @param array $events + * @param string $payload + * @param Document $webhook + * @param Document $user + * @param Document $project + * @return void + */ + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project): void + { + + $url = \rawurldecode($webhook->getAttribute('url')); + $signatureKey = $webhook->getAttribute('signatureKey'); + $signature = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); + $httpUser = $webhook->getAttribute('httpUser'); + $httpPass = $webhook->getAttribute('httpPass'); + $ch = \curl_init($webhook->getAttribute('url')); + + \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); + \curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + \curl_setopt($ch, CURLOPT_HEADER, 0); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + \curl_setopt($ch, CURLOPT_USERAGENT, \sprintf( + APP_USERAGENT, + App::getEnv('_APP_VERSION', 'UNKNOWN'), + App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY) + )); + \curl_setopt( + $ch, + CURLOPT_HTTPHEADER, + [ + 'Content-Type: application/json', + 'Content-Length: ' . \strlen($payload), + 'X-' . APP_NAME . '-Webhook-Id: ' . $webhook->getId(), + 'X-' . APP_NAME . '-Webhook-Events: ' . implode(',', $events), + 'X-' . APP_NAME . '-Webhook-Name: ' . $webhook->getAttribute('name', ''), + 'X-' . APP_NAME . '-Webhook-User-Id: ' . $user->getId(), + 'X-' . APP_NAME . '-Webhook-Project-Id: ' . $project->getId(), + 'X-' . APP_NAME . '-Webhook-Signature: ' . $signature, + ] + ); + + if (!$webhook->getAttribute('security', true)) { + \curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); + \curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + } + + if (!empty($httpUser) && !empty($httpPass)) { + \curl_setopt($ch, CURLOPT_USERPWD, "$httpUser:$httpPass"); + \curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + } + + if (false === \curl_exec($ch)) { + $this->errors[] = \curl_error($ch) . ' in events ' . implode(', ', $events) . ' for webhook ' . $webhook->getAttribute('name'); + } + + \curl_close($ch); + } +} From 2884cb636b7803fd075150b15e7b940c347bdb73 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 21 Oct 2023 07:18:21 -0400 Subject: [PATCH 004/406] cleanup --- app/console | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/console b/app/console index e965738987..9b4bcb8140 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit e9657389879c8d76a9b3a0d3486c1d86f43c3bb9 +Subproject commit 9b4bcb8140484669421685b4ba89fa1c4d331360 From 77a43006690c2dab46b7f059219621a11f419c0b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 21 Oct 2023 07:22:46 -0400 Subject: [PATCH 005/406] cleanups --- package 2.json | 8 -------- package-lock.json | 10 ---------- pnpm-lock.yaml | 5 ----- 3 files changed, 23 deletions(-) delete mode 100644 package 2.json delete mode 100644 package-lock.json delete mode 100644 pnpm-lock.yaml diff --git a/package 2.json b/package 2.json deleted file mode 100644 index 6e32c7d515..0000000000 --- a/package 2.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "private": true, - "name": "@appwrite.io/repo", - "repository": { - "type": "git", - "url": "git+https://github.com/appwrite/appwrite.git" - } -} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 6ab33b14fe..0000000000 --- a/package-lock.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "@appwrite.io/repo", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@appwrite.io/repo" - } - } -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml deleted file mode 100644 index 2b9f1883a1..0000000000 --- a/pnpm-lock.yaml +++ /dev/null @@ -1,5 +0,0 @@ -lockfileVersion: '6.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false From 92538c57f6570712907cfc8a015a13e5d34a3004 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 21 Oct 2023 07:29:23 -0400 Subject: [PATCH 006/406] cleanups --- app/test | 184 ------------------------------------------------------- 1 file changed, 184 deletions(-) delete mode 100644 app/test diff --git a/app/test b/app/test deleted file mode 100644 index a4d28b1200..0000000000 --- a/app/test +++ /dev/null @@ -1,184 +0,0 @@ -$register->set('pools', function () { - $group = new Group(); - - $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ - 'scheme' => 'mariadb', - 'host' => App::getEnv('_APP_DB_HOST', 'mariadb'), - 'port' => App::getEnv('_APP_DB_PORT', '3306'), - 'user' => App::getEnv('_APP_DB_USER', ''), - 'pass' => App::getEnv('_APP_DB_PASS', ''), - 'path' => App::getEnv('_APP_DB_SCHEMA', ''), - ]); - $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ - 'scheme' => 'redis', - 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), - 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), - 'user' => App::getEnv('_APP_REDIS_USER', ''), - 'pass' => App::getEnv('_APP_REDIS_PASS', ''), - ]); - - $connections = [ - 'console' => [ - 'type' => 'database', - 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), - 'multiple' => false, - 'schemes' => ['mariadb', 'mysql'], - ], - 'database' => [ - 'type' => 'database', - 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), - 'multiple' => true, - 'schemes' => ['mariadb', 'mysql'], - ], - 'queue' => [ - 'type' => 'queue', - 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), - 'multiple' => false, - 'schemes' => ['redis'], - ], - 'pubsub' => [ - 'type' => 'pubsub', - 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), - 'multiple' => false, - 'schemes' => ['redis'], - ], - 'cache' => [ - 'type' => 'cache', - 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), - 'multiple' => true, - 'schemes' => ['redis'], - ], - ]; - - $maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151); - $instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14); - - $multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; - - if ($multiprocessing) { - $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); - } else { - $workerCount = 1; - } - - if ($workerCount > $instanceConnections) { - throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); - } - - $poolSize = (int)($instanceConnections / $workerCount); - - foreach ($connections as $key => $connection) { - $type = $connection['type'] ?? ''; - $dsns = $connection['dsns'] ?? ''; - $multipe = $connection['multiple'] ?? false; - $schemes = $connection['schemes'] ?? []; - $config = []; - $dsns = explode(',', $connection['dsns'] ?? ''); - foreach ($dsns as &$dsn) { - $dsn = explode('=', $dsn); - $name = ($multipe) ? $key . '_' . $dsn[0] : $key; - $dsn = $dsn[1] ?? ''; - $config[] = $name; - if (empty($dsn)) { - //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); - continue; - } - - $dsn = new DSN($dsn); - $dsnHost = $dsn->getHost(); - $dsnPort = $dsn->getPort(); - $dsnUser = $dsn->getUser(); - $dsnPass = $dsn->getPassword(); - $dsnScheme = $dsn->getScheme(); - $dsnDatabase = $dsn->getPath(); - - if (!in_array($dsnScheme, $schemes)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); - } - - /** - * Get Resource - * - * Creation could be reused accross connection types like database, cache, queue, etc. - * - * Resource assignment to an adapter will happen below. - */ - switch ($dsnScheme) { - case 'mysql': - case 'mariadb': - $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - )); - }); - }; - break; - case 'redis': - $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); - @$redis->pconnect($dsnHost, (int)$dsnPort); - if ($dsnPass) { - $redis->auth($dsnPass); - } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; - }; - break; - - default: - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); - break; - } - - $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { - // Get Adapter - $adapter = null; - switch ($type) { - case 'database': - $adapter = match ($dsn->getScheme()) { - 'mariadb' => new MariaDB($resource()), - 'mysql' => new MySQL($resource()), - default => null - }; - - $adapter->setDefaultDatabase($dsn->getPath()); - break; - case 'pubsub': - $adapter = $resource(); - break; - case 'queue': - $adapter = match ($dsn->getScheme()) { - 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), - default => null - }; - break; - case 'cache': - $adapter = match ($dsn->getScheme()) { - 'redis' => new RedisCache($resource()), - default => null - }; - break; - - default: - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); - break; - } - - return $adapter; - }); - - $group->add($pool); - } - - Config::setParam('pools-' . $key, $config); - } - - return $group; -}); From f4fedc832e96e978a0d98f1353e9436cb6754d7d Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 21 Oct 2023 08:30:35 -0400 Subject: [PATCH 007/406] Fix formating --- app/http.php | 12 ++++++------ app/init.php | 48 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/app/http.php b/app/http.php index 72b89130b9..0eadb5c1e1 100644 --- a/app/http.php +++ b/app/http.php @@ -333,19 +333,19 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $connectionForQueue = $app->getResource('connectionForQueue'); $connectionsForCache = $app->getResource('connectionsForCache'); - if(!is_null($connectionForConsole)) { + if (!is_null($connectionForConsole)) { $connectionForConsole->reclaim(); } - - if(!is_null($connectionForProject)) { + + if (!is_null($connectionForProject)) { $connectionForProject->reclaim(); } - - if(!is_null($connectionForQueue)) { + + if (!is_null($connectionForQueue)) { $connectionForQueue->reclaim(); } - if(!empty($connectionsForCache)) { + if (!empty($connectionsForCache)) { foreach ($connectionsForCache as $connection) { $connection->reclaim(); } diff --git a/app/init.php b/app/init.php index 77fd558a56..f73879a02b 100644 --- a/app/init.php +++ b/app/init.php @@ -885,8 +885,10 @@ App::setResource('localeCodes', function () { // Queues App::setResource('queue', function (Group $pools) { $connection = $pools->get('queue')->pop(); - - App::setResource('connectionForQueue', function () use ($connection) { return $connection; }, []); + + App::setResource('connectionForQueue', function () use ($connection) { + return $connection; + }, []); return $connection->getResource(); }, ['pools']); @@ -1112,10 +1114,18 @@ App::setResource('console', function () { ]); }, []); -App::setResource('connectionForProject', function () { return null; }, []); -App::setResource('connectionForConsole', function () { return null; }, []); -App::setResource('connectionForQueue', function () { return null; }, []); -App::setResource('connectionsForCache', function () { return []; }, []); +App::setResource('connectionForProject', function () { + return null; +}, []); +App::setResource('connectionForConsole', function () { + return null; +}, []); +App::setResource('connectionForQueue', function () { + return null; +}, []); +App::setResource('connectionsForCache', function () { + return []; +}, []); App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { @@ -1127,10 +1137,12 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, ->pop() ; - App::setResource('connectionForProject', function () use ($connection) { return $connection; }, []); - + App::setResource('connectionForProject', function () use ($connection) { + return $connection; + }, []); + $dbAdapter = $connection->getResource(); - + $database = new Database($dbAdapter, $cache); $database @@ -1144,9 +1156,11 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $connection = $pools ->get('console') ->pop(); - - App::setResource('connectionForConsole', function () use ($connection) { return $connection; }, []); - + + App::setResource('connectionForConsole', function () use ($connection) { + return $connection; + }, []); + $dbAdapter = $connection->getResource(); $database = new Database($dbAdapter, $cache); @@ -1163,7 +1177,7 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } - + $connection = $pools ->get($project->getAttribute('database')) ->pop() @@ -1171,7 +1185,9 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $dbAdapter = $connection->getResource(); - App::setResource('connectionForProject', function () use ($connection) { return $connection; }, []); + App::setResource('connectionForProject', function () use ($connection) { + return $connection; + }, []); $database = new Database($dbAdapter, $cache); @@ -1200,7 +1216,9 @@ App::setResource('cache', function (Group $pools) { $adapters[] = $connection->getResource(); } - App::setResource('connectionsForCache', function () use ($connections) { return $connections; }, []); + App::setResource('connectionsForCache', function () use ($connections) { + return $connections; + }, []); return new Cache(new Sharding($adapters)); }, ['pools']); From 464b913e90bd2dcf62b22334902dcf4a287d1ade Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Nov 2023 21:32:36 +1300 Subject: [PATCH 008/406] Update for function name change --- app/cli.php | 2 +- app/http.php | 2 +- app/init.php | 44 ++++++++----------- appwrite.json | 4 ++ src/Appwrite/Migration/Migration.php | 2 +- src/Appwrite/Migration/Version/V15.php | 18 ++++---- src/Appwrite/Migration/Version/V18.php | 4 +- src/Appwrite/Migration/Version/V19.php | 2 +- src/Appwrite/Platform/Tasks/CalcTierStats.php | 2 +- .../Platform/Tasks/DeleteOrphanedProjects.php | 6 +-- src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 11 files changed, 42 insertions(+), 46 deletions(-) create mode 100644 appwrite.json diff --git a/app/cli.php b/app/cli.php index 643a615c46..d3e44ea398 100644 --- a/app/cli.php +++ b/app/cli.php @@ -71,7 +71,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { $collections = Config::getParam('collections', [])['console']; $last = \array_key_last($collections); - if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { /** TODO cache ready variable using registry */ + if (!($dbForConsole->exists($dbForConsole->getDatabase(), $last))) { /** TODO cache ready variable using registry */ throw new Exception('Tables not ready yet.'); } diff --git a/app/http.php b/app/http.php index fe1ed48724..781df53e98 100644 --- a/app/http.php +++ b/app/http.php @@ -147,7 +147,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection($key, $attributes, $indexes); } - if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) { + if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDatabase(), 'bucket_1')) { Console::success('[Setup] - Creating default bucket...'); $dbForConsole->createDocument('buckets', new Document([ '$id' => ID::custom('default'), diff --git a/app/init.php b/app/init.php index 2c0219eec2..69dcdc737b 100644 --- a/app/init.php +++ b/app/init.php @@ -12,11 +12,11 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { require_once __DIR__ . '/../vendor/autoload.php'; } -ini_set('memory_limit', '512M'); -ini_set('display_errors', 1); -ini_set('display_startup_errors', 1); -ini_set('default_socket_timeout', -1); -error_reporting(E_ALL); +\ini_set('memory_limit', '512M'); +\ini_set('display_errors', 1); +\ini_set('display_startup_errors', 1); +\ini_set('default_socket_timeout', -1); +\error_reporting(E_ALL); use Appwrite\Event\Migration; use Appwrite\Extend\Exception; @@ -34,6 +34,7 @@ use Appwrite\OpenSSL\OpenSSL; use Appwrite\URL\URL as AppwriteURL; use Appwrite\Usage\Stats; use Utopia\App; +use Utopia\CLI\Console; use Utopia\Logger\Logger; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Cache; @@ -653,14 +654,13 @@ $register->set('pools', function () { foreach ($connections as $key => $connection) { $type = $connection['type'] ?? ''; - $dsns = $connection['dsns'] ?? ''; - $multipe = $connection['multiple'] ?? false; + $multiple = $connection['multiple'] ?? false; $schemes = $connection['schemes'] ?? []; $config = []; $dsns = explode(',', $connection['dsns'] ?? ''); foreach ($dsns as &$dsn) { $dsn = explode('=', $dsn); - $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $name = ($multiple) ? $key . '_' . $dsn[0] : $key; $dsn = $dsn[1] ?? ''; $config[] = $name; if (empty($dsn)) { @@ -732,7 +732,7 @@ $register->set('pools', function () { default => null }; - $adapter->setDefaultDatabase($dsn->getPath()); + $adapter->setDatabase($dsn->getPath()); break; case 'pubsub': $adapter = $resource(); @@ -1136,8 +1136,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $dbAdapter = $pools ->get('console') ->pop() - ->getResource() - ; + ->getResource(); $database = new Database($dbAdapter, $cache); @@ -1247,21 +1246,14 @@ function getDevice($root): Device Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); } - switch ($device) { - case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE::DEVICE_DO_SPACES: - return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_BACKBLAZE: - return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LINODE: - return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_WASABI: - return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LOCAL: - default: - return new Local($root); - } + return match ($device) { + Storage::DEVICE_S3 => new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl), + STORAGE::DEVICE_DO_SPACES => new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl), + Storage::DEVICE_BACKBLAZE => new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl), + Storage::DEVICE_LINODE => new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl), + Storage::DEVICE_WASABI => new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl), + default => new Local($root), + }; } else { switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) { case Storage::DEVICE_LOCAL: diff --git a/appwrite.json b/appwrite.json new file mode 100644 index 0000000000..e3d948ed1a --- /dev/null +++ b/appwrite.json @@ -0,0 +1,4 @@ +{ + "projectId": "console", + "projectName": "" +} \ No newline at end of file diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 8f68e31be4..de26941a5a 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -388,7 +388,7 @@ abstract class Migration */ protected function changeAttributeInternalType(string $collection, string $attribute, string $type): void { - $stmt = $this->pdo->prepare("ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collection}` MODIFY `$attribute` $type;"); + $stmt = $this->pdo->prepare("ALTER TABLE `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_{$collection}` MODIFY `$attribute` $type;"); try { $stmt->execute(); diff --git a/src/Appwrite/Migration/Version/V15.php b/src/Appwrite/Migration/Version/V15.php index 60f5fa20ab..9d8af4482a 100644 --- a/src/Appwrite/Migration/Version/V15.php +++ b/src/Appwrite/Migration/Version/V15.php @@ -295,7 +295,7 @@ class V15 extends Migration protected function removeWritePermissions(string $table): void { try { - $this->pdo->prepare("DELETE FROM `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}_perms` WHERE _type = 'write'")->execute(); + $this->pdo->prepare("DELETE FROM `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_{$table}_perms` WHERE _type = 'write'")->execute(); } catch (\Throwable $th) { Console::warning("Remove 'write' permissions from {$table}: {$th->getMessage()}"); } @@ -311,7 +311,7 @@ class V15 extends Migration */ protected function getSQLColumnTypes(string $table): array { - $query = $this->pdo->prepare("SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '_{$this->project->getInternalId()}_{$table}' AND table_schema = '{$this->projectDB->getDefaultDatabase()}'"); + $query = $this->pdo->prepare("SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '_{$this->project->getInternalId()}_{$table}' AND table_schema = '{$this->projectDB->getDatabase()}'"); $query->execute(); return array_reduce($query->fetchAll(), function (array $carry, array $item) { @@ -333,8 +333,8 @@ class V15 extends Migration if ($columns[$attribute] === 'int') { try { - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` MODIFY {$attribute} VARCHAR(64)")->execute(); - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` SET {$attribute} = IF({$attribute} = 0, NULL, FROM_UNIXTIME({$attribute}))")->execute(); + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_{$table}` MODIFY {$attribute} VARCHAR(64)")->execute(); + $this->pdo->prepare("UPDATE `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_{$table}` SET {$attribute} = IF({$attribute} = 0, NULL, FROM_UNIXTIME({$attribute}))")->execute(); $columns[$attribute] = 'varchar'; } catch (\Throwable $th) { Console::warning($th->getMessage()); @@ -343,7 +343,7 @@ class V15 extends Migration if ($columns[$attribute] === 'varchar') { try { - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` MODIFY {$attribute} DATETIME(3)")->execute(); + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_{$table}` MODIFY {$attribute} DATETIME(3)")->execute(); } catch (\Throwable $th) { Console::warning($th->getMessage()); } @@ -389,7 +389,7 @@ class V15 extends Migration if (!array_key_exists('_permissions', $columns)) { try { - $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}` ADD `_permissions` MEDIUMTEXT DEFAULT NULL")->execute(); + $this->pdo->prepare("ALTER TABLE IF EXISTS `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_{$table}` ADD `_permissions` MEDIUMTEXT DEFAULT NULL")->execute(); } catch (\Throwable $th) { Console::warning("Add '_permissions' column to '{$table}': {$th->getMessage()}"); } @@ -410,7 +410,7 @@ class V15 extends Migration { $table ??= $document->getCollection(); - $query = $this->pdo->prepare("SELECT * FROM `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$table}_perms` WHERE _document = '{$document->getId()}'"); + $query = $this->pdo->prepare("SELECT * FROM `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_{$table}_perms` WHERE _document = '{$document->getId()}'"); $query->execute(); $results = $query->fetchAll(); $permissions = []; @@ -1472,9 +1472,9 @@ class V15 extends Migration $from = $this->pdo->quote($from); $to = $this->pdo->quote($to); - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_stats` SET metric = {$to} WHERE metric = {$from}")->execute(); + $this->pdo->prepare("UPDATE `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_stats` SET metric = {$to} WHERE metric = {$from}")->execute(); } catch (\Throwable $th) { - Console::warning("Migrating steps from {$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_stats:" . $th->getMessage()); + Console::warning("Migrating steps from {$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}_stats:" . $th->getMessage()); } } diff --git a/src/Appwrite/Migration/Version/V18.php b/src/Appwrite/Migration/Version/V18.php index 839269f940..12d573e549 100644 --- a/src/Appwrite/Migration/Version/V18.php +++ b/src/Appwrite/Migration/Version/V18.php @@ -244,7 +244,7 @@ class V18 extends Migration /** * Create 'documentSecurity' column */ - $this->pdo->prepare("ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` ADD COLUMN IF NOT EXISTS documentSecurity TINYINT(1);")->execute(); + $this->pdo->prepare("ALTER TABLE `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}__metadata` ADD COLUMN IF NOT EXISTS documentSecurity TINYINT(1);")->execute(); } catch (\Throwable $th) { Console::warning($th->getMessage()); } @@ -253,7 +253,7 @@ class V18 extends Migration /** * Set 'documentSecurity' column to 1 if NULL */ - $this->pdo->prepare("UPDATE `{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}__metadata` SET documentSecurity = 1 WHERE documentSecurity IS NULL")->execute(); + $this->pdo->prepare("UPDATE `{$this->projectDB->getDatabase()}`.`_{$this->project->getInternalId()}__metadata` SET documentSecurity = 1 WHERE documentSecurity IS NULL")->execute(); } catch (\Throwable $th) { Console::warning($th->getMessage()); } diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index b9ccf3c302..c04a3b03de 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -50,7 +50,7 @@ class V19 extends Migration protected function migrateDomains(): void { - if ($this->consoleDB->exists($this->consoleDB->getDefaultDatabase(), 'domains')) { + if ($this->consoleDB->exists($this->consoleDB->getDatabase(), 'domains')) { foreach ($this->documentsIterator('domains') as $domain) { $status = 'created'; if ($domain->getAttribute('verification', false)) { diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 2a2bc20af9..e5a9550666 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -122,7 +122,7 @@ class CalcTierStats extends Action ->getResource(); $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); /** Get Project ID */ diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 757b29c1b6..5a04df04ba 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -90,12 +90,12 @@ class DeleteOrphanedProjects extends Action ->getResource(); $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = 0; $cnt++; - if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + if ($dbForProject->exists($dbForProject->getDatabase(), Database::METADATA)) { $collectionsCreated = $dbForProject->count(Database::METADATA); } @@ -123,7 +123,7 @@ class DeleteOrphanedProjects extends Action $dbForConsole->deleteDocument('projects', $project->getId()); $dbForConsole->deleteCachedDocument('projects', $project->getId()); - if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + if ($dbForProject->exists($dbForProject->getDatabase(), Database::METADATA)) { try { $dbForProject->deleteCollection(Database::METADATA); $dbForProject->deleteCachedCollection(Database::METADATA); diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 1d5d3b0b26..0947da65d9 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -81,7 +81,7 @@ class Hamster extends Action ->getResource(); $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $statsPerProject = []; From eb5fc0797fc4b190d1668b6cd6582078aa6e81c8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Nov 2023 00:43:32 +1300 Subject: [PATCH 009/406] Update abuse/audit --- composer.json | 6 ++--- composer.lock | 69 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/composer.json b/composer.json index a24feca83b..0d62b47988 100644 --- a/composer.json +++ b/composer.json @@ -43,13 +43,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.13.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.33.*", + "utopia-php/abuse": "dev-feat-isolation-modes as 0.33.0", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.35.*", + "utopia-php/audit": "dev-feat-isolation-modes as 0.35.0", "utopia-php/cache": "0.8.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.45.*", + "utopia-php/database": "dev-feat-isolation-mode as 0.45.2", "utopia-php/domains": "0.3.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.31.0", diff --git a/composer.lock b/composer.lock index d362f3aa9c..3f9bf551d2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "69bc2e21a65b78344393706b39d789b4", + "content-hash": "2c23008af30193734b6717a41632aa54", "packages": [ { "name": "adhocore/jwt", @@ -1615,23 +1615,23 @@ }, { "name": "utopia-php/abuse", - "version": "0.33.0", + "version": "dev-feat-isolation-modes", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "1ba8d5f2793885cbf779e3b5b9d886968af43d2c" + "reference": "171eda04bfc53e5e24bdb36230ae84fe686ac2ee" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/1ba8d5f2793885cbf779e3b5b9d886968af43d2c", - "reference": "1ba8d5f2793885cbf779e3b5b9d886968af43d2c", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/171eda04bfc53e5e24bdb36230ae84fe686ac2ee", + "reference": "171eda04bfc53e5e24bdb36230ae84fe686ac2ee", "shasum": "" }, "require": { "ext-curl": "*", "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.45.*" + "utopia-php/database": "dev-feat-isolation-mode as 0.45.0" }, "require-dev": { "laravel/pint": "1.5.*", @@ -1658,9 +1658,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.33.0" + "source": "https://github.com/utopia-php/abuse/tree/feat-isolation-modes" }, - "time": "2023-11-01T08:51:33+00:00" + "time": "2023-11-22T09:49:43+00:00" }, { "name": "utopia-php/analytics", @@ -1710,21 +1710,21 @@ }, { "name": "utopia-php/audit", - "version": "0.35.0", + "version": "dev-feat-isolation-modes", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "ed9366ef05556da040de7a8b570f4160c7d8ea4a" + "reference": "b561fa872c60e28afda02b4428ad5a2444099c44" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/ed9366ef05556da040de7a8b570f4160c7d8ea4a", - "reference": "ed9366ef05556da040de7a8b570f4160c7d8ea4a", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/b561fa872c60e28afda02b4428ad5a2444099c44", + "reference": "b561fa872c60e28afda02b4428ad5a2444099c44", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.45.*" + "utopia-php/database": "dev-feat-isolation-mode as 0.45.0" }, "require-dev": { "laravel/pint": "1.5.*", @@ -1751,9 +1751,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.35.0" + "source": "https://github.com/utopia-php/audit/tree/feat-isolation-modes" }, - "time": "2023-11-01T08:51:29+00:00" + "time": "2023-11-22T09:49:57+00:00" }, { "name": "utopia-php/cache", @@ -1906,16 +1906,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.2", + "version": "dev-feat-isolation-mode", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788" + "reference": "b7bcc9b373242a9e494b0b7bbf78dfa54333727c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/dc789f2c1fd8b5ee07ff883e11c9ad7970824788", - "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b7bcc9b373242a9e494b0b7bbf78dfa54333727c", + "reference": "b7bcc9b373242a9e494b0b7bbf78dfa54333727c", "shasum": "" }, "require": { @@ -1956,9 +1956,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.2" + "source": "https://github.com/utopia-php/database/tree/feat-isolation-mode" }, - "time": "2023-11-15T03:38:47+00:00" + "time": "2023-11-22T07:58:21+00:00" }, { "name": "utopia-php/domains", @@ -5796,9 +5796,32 @@ "time": "2023-08-28T11:09:02+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/abuse", + "version": "dev-feat-isolation-modes", + "alias": "0.33.0", + "alias_normalized": "0.33.0.0" + }, + { + "package": "utopia-php/audit", + "version": "dev-feat-isolation-modes", + "alias": "0.35.0", + "alias_normalized": "0.35.0.0" + }, + { + "package": "utopia-php/database", + "version": "dev-feat-isolation-mode", + "alias": "0.45.2", + "alias_normalized": "0.45.2.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/abuse": 20, + "utopia-php/audit": 20, + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From cefc1b6519e8f34e7aeb71d2d5c6704f8117b3e3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Nov 2023 01:11:48 +1300 Subject: [PATCH 010/406] Add base share tables check when creating project --- app/controllers/api/projects.php | 64 +++++++++++++++++--------------- app/init.php | 5 ++- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index b8f8ac4727..eb41728c6e 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -77,7 +77,6 @@ App::post('/v1/projects') ->inject('pools') ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { - $team = $dbForConsole->getDocument('teams', $teamId); if ($team->isEmpty()) { @@ -85,13 +84,22 @@ App::post('/v1/projects') } $auth = Config::getParam('auth', []); - $auths = ['limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false]; - foreach ($auth as $index => $method) { + $auths = [ + 'limit' => 0, + 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, + 'passwordHistory' => 0, + 'passwordDictionary' => false, + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, + 'personalDataCheck' => false + ]; + foreach ($auth as $method) { $auths[$method['key'] ?? ''] = true; } $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId; + $dbForConsole->setTenant($projectId); + $backups['database_db_fra1_v14x_02'] = ['from' => '03:00', 'to' => '05:00']; $backups['database_db_fra1_v14x_03'] = ['from' => '00:00', 'to' => '02:00']; $backups['database_db_fra1_v14x_04'] = ['from' => '00:00', 'to' => '02:00']; @@ -133,6 +141,9 @@ App::post('/v1/projects') throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); } + static $counter = 0; + $shareTables = $counter++ % 2 === 0; + try { $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, @@ -166,12 +177,21 @@ App::post('/v1/projects') 'search' => implode(' ', [$projectId, $name]), 'database' => $database ])); - } catch (Duplicate $th) { + } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); } $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); - $dbForProject->setNamespace("_{$project->getInternalId()}"); + + if ($shareTables) { + $dbForProject + ->setShareTables(true) + ->setTenant($project->getId()); + } else { + $dbForProject + ->setNamespace("_{$project->getInternalId()}"); + } + $dbForProject->create(); $audit = new Audit($dbForProject); @@ -188,33 +208,19 @@ App::post('/v1/projects') continue; } - $attributes = []; - $indexes = []; + $attributes = \array_map(function (array $attribute) { + return new Document($attribute); + }, $collection['attributes']); - foreach ($collection['attributes'] as $attribute) { - $attributes[] = new Document([ - '$id' => $attribute['$id'], - 'type' => $attribute['type'], - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'array' => $attribute['array'], - 'filters' => $attribute['filters'], - 'default' => $attribute['default'] ?? null, - 'format' => $attribute['format'] ?? '' - ]); - } + $indexes = \array_map(function (array $index) { + return new Document($index); + }, $collection['indexes']); - foreach ($collection['indexes'] as $index) { - $indexes[] = new Document([ - '$id' => $index['$id'], - 'type' => $index['type'], - 'attributes' => $index['attributes'], - 'lengths' => $index['lengths'], - 'orders' => $index['orders'], - ]); + try { + $dbForProject->createCollection($key, $attributes, $indexes); + } catch (Duplicate) { + // Collection already exists } - $dbForProject->createCollection($key, $attributes, $indexes); } $response diff --git a/app/init.php b/app/init.php index 69dcdc737b..035952d77f 100644 --- a/app/init.php +++ b/app/init.php @@ -1132,7 +1132,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, return $database; }, ['pools', 'dbForConsole', 'cache', 'project']); -App::setResource('dbForConsole', function (Group $pools, Cache $cache) { +App::setResource('dbForConsole', function (Group $pools, Cache $cache, Document $project) { $dbAdapter = $pools ->get('console') ->pop() @@ -1141,13 +1141,14 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $database = new Database($dbAdapter, $cache); $database + ->setTenant($project->getId()) ->setNamespace('_console') ->setMetadata('host', \gethostname()) ->setMetadata('project', 'console') ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); return $database; -}, ['pools', 'cache']); +}, ['pools', 'cache', 'project']); App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools From f44543f083784d812b86434fb36436296b5238ff Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Nov 2023 01:12:19 +1300 Subject: [PATCH 011/406] Add shareTables attribute to project --- app/config/collections.php | 11 +++++++++++ app/controllers/api/projects.php | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/config/collections.php b/app/config/collections.php index db229ce87a..60eeaf40a8 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3268,6 +3268,17 @@ $consoleCollections = array_merge([ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('shareTables'), + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index eb41728c6e..8461fb2777 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -175,7 +175,8 @@ App::post('/v1/projects') 'keys' => null, 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), - 'database' => $database + 'database' => $database, + 'shareTables' => $shareTables, ])); } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); From 35163ff52a092765a5024e864cc5e9c8ae8b5e78 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Nov 2023 01:13:24 +1300 Subject: [PATCH 012/406] Check project shareTables status when configuring databases --- app/cli.php | 2 +- app/init.php | 35 +++++++++++++++++++++-------------- app/worker.php | 40 ++++++++++++++++++++++++++++++++-------- 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/app/cli.php b/app/cli.php index d3e44ea398..c341c4849c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -24,7 +24,7 @@ use Utopia\Registry\Registry; Authorization::disable(); -CLI::setResource('register', fn()=>$register); +CLI::setResource('register', fn() => $register); CLI::setResource('cache', function ($pools) { $list = Config::getParam('pools-cache', []); diff --git a/app/init.php b/app/init.php index 035952d77f..a8e205f1ee 100644 --- a/app/init.php +++ b/app/init.php @@ -1129,6 +1129,13 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + if ($project->getAttribute('shareTables')) { + $database + ->setNamespace('') + ->setShareTables(true) + ->setTenant($project->getId()); + } + return $database; }, ['pools', 'dbForConsole', 'cache', 'project']); @@ -1153,22 +1160,31 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache, Document App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { + return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } $databaseName = $project->getAttribute('database'); - if (isset($databases[$databaseName])) { - $database = $databases[$databaseName]; - + $configure = (function (Database $database) use ($project) { $database ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + if ($project->getAttribute('shareTables')) { + $database + ->setNamespace('') + ->setShareTables(true) + ->setTenant($project->getId()); + } + }); + + if (isset($databases[$databaseName])) { + $database = $databases[$databaseName]; + $configure($database); return $database; } @@ -1178,19 +1194,10 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->getResource(); $database = new Database($dbAdapter, $cache); - $databases[$databaseName] = $database; - - $database - ->setNamespace('_' . $project->getInternalId()) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - + $configure($database); return $database; }; - - return $getProjectDB; }, ['pools', 'dbForConsole', 'cache']); App::setResource('cache', function (Group $pools) { diff --git a/app/worker.php b/app/worker.php index 32a8b9804e..98dc9e682e 100644 --- a/app/worker.php +++ b/app/worker.php @@ -61,15 +61,25 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, } $pools = $register->get('pools'); - $database = $pools + + $adapter = $pools ->get($project->getAttribute('database')) ->pop() - ->getResource() - ; + ->getResource(); - $adapter = new Database($database, $cache); - $adapter->setNamespace('_' . $project->getInternalId()); - return $adapter; + $database = new Database($adapter, $cache); + + if ($project->getAttribute('shareTables')) { + $database + ->setShareTables(true) + ->setTenant($project->getId()); + } else { + $database + ->setNamespace('_' . $project->getInternalId()); + } + + + return $database; }, ['cache', 'register', 'message', 'dbForConsole']); Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { @@ -84,7 +94,14 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; - $database->setNamespace('_' . $project->getInternalId()); + if ($project->getAttribute('shareTables')) { + $database + ->setShareTables(true) + ->setTenant($project->getId()); + } else { + $database + ->setNamespace('_' . $project->getInternalId()); + } return $database; } @@ -97,7 +114,14 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $databases[$databaseName] = $database; - $database->setNamespace('_' . $project->getInternalId()); + if ($project->getAttribute('shareTables')) { + $database + ->setShareTables(true) + ->setTenant($project->getId()); + } else { + $database + ->setNamespace('_' . $project->getInternalId()); + } return $database; }; From 7b8e37b03d757507bdea9c8670f7e7d4248257dd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 27 Nov 2023 15:22:05 +1300 Subject: [PATCH 013/406] Update method names --- app/controllers/api/account.php | 38 ++++++++-------- app/controllers/api/avatars.php | 2 +- app/controllers/api/databases.php | 38 ++++++++-------- app/controllers/api/projects.php | 20 ++++----- app/controllers/api/teams.php | 12 ++--- app/controllers/api/users.php | 4 +- app/controllers/general.php | 9 ++-- app/controllers/shared/api.php | 2 +- app/init.php | 29 ++++++------ composer.lock | 8 ++-- src/Appwrite/Migration/Version/V15.php | 4 +- src/Appwrite/Migration/Version/V17.php | 32 +++++++------- src/Appwrite/Migration/Version/V18.php | 6 +-- src/Appwrite/Migration/Version/V19.php | 44 +++++++++---------- .../Platform/Tasks/DeleteOrphanedProjects.php | 6 +-- src/Appwrite/Platform/Workers/Databases.php | 20 ++++----- src/Appwrite/Platform/Workers/Deletes.php | 8 ++-- src/Appwrite/Platform/Workers/Migrations.php | 2 +- 18 files changed, 145 insertions(+), 139 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index c210b19f4f..175c759bbc 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -248,7 +248,7 @@ App::post('/v1/account/sessions/email') $dbForProject->updateDocument('users', $user->getId(), $user); } - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $session = $dbForProject->createDocument('sessions', $session->setAttribute('$permissions', [ Permission::read(Role::user($user->getId())), @@ -567,7 +567,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $currentDocument = $dbForProject->getDocument('sessions', $current); if (!$currentDocument->isEmpty()) { $dbForProject->deleteDocument('sessions', $currentDocument->getId()); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); } } @@ -755,7 +755,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $session->setAttribute('expire', $expire); @@ -985,7 +985,7 @@ App::post('/v1/account/sessions/magic-url') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); if (empty($url)) { $url = $request->getProtocol() . '://' . $request->getHostname() . '/auth/magic-url'; @@ -1164,7 +1164,7 @@ App::put('/v1/account/sessions/magic-url') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $tokens = $user->getAttribute('tokens', []); @@ -1173,7 +1173,7 @@ App::put('/v1/account/sessions/magic-url') * the recovery token but actually we don't need it anymore. */ $dbForProject->deleteDocument('tokens', $token); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $user->setAttribute('emailVerification', true); @@ -1311,7 +1311,7 @@ App::post('/v1/account/sessions/phone') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/sms-base.tpl'); @@ -1415,14 +1415,14 @@ App::put('/v1/account/sessions/phone') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); /** * We act like we're updating and validating * the recovery token but actually we don't need it anymore. */ $dbForProject->deleteDocument('tokens', $token); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $user->setAttribute('phoneVerification', true); @@ -1569,7 +1569,7 @@ App::post('/v1/account/sessions/anonymous') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $queueForEvents ->setParam('userId', $user->getId()) @@ -2203,7 +2203,7 @@ App::delete('/v1/account/sessions/:sessionId') ; } - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $queueForEvents ->setParam('userId', $user->getId()) @@ -2284,7 +2284,7 @@ App::patch('/v1/account/sessions/:sessionId') $dbForProject->updateDocument('sessions', $sessionId, $session); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; @@ -2355,7 +2355,7 @@ App::delete('/v1/account/sessions') } } - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $queueForEvents ->setParam('userId', $user->getId()) @@ -2441,7 +2441,7 @@ App::post('/v1/account/recovery') Permission::delete(Role::user($profile->getId())), ])); - $dbForProject->deleteCachedDocument('users', $profile->getId()); + $dbForProject->purgeCachedDocument('users', $profile->getId()); $url = Template::parseURL($url); $url['query'] = Template::mergeQuery(((isset($url['query'])) ? $url['query'] : ''), ['userId' => $profile->getId(), 'secret' => $secret, 'expire' => $expire]); @@ -2620,7 +2620,7 @@ App::put('/v1/account/recovery') * the recovery token but actually we don't need it anymore. */ $dbForProject->deleteDocument('tokens', $recovery); - $dbForProject->deleteCachedDocument('users', $profile->getId()); + $dbForProject->purgeCachedDocument('users', $profile->getId()); $queueForEvents ->setParam('userId', $profile->getId()) @@ -2692,7 +2692,7 @@ App::post('/v1/account/verification') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $url = Template::parseURL($url); $url['query'] = Template::mergeQuery(((isset($url['query'])) ? $url['query'] : ''), ['userId' => $user->getId(), 'secret' => $verificationSecret, 'expire' => $expire]); @@ -2842,7 +2842,7 @@ App::put('/v1/account/verification') * the verification token but actually we don't need it anymore. */ $dbForProject->deleteDocument('tokens', $verification); - $dbForProject->deleteCachedDocument('users', $profile->getId()); + $dbForProject->purgeCachedDocument('users', $profile->getId()); $queueForEvents ->setParam('userId', $userId) @@ -2917,7 +2917,7 @@ App::post('/v1/account/verification/phone') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/sms-base.tpl'); @@ -3001,7 +3001,7 @@ App::put('/v1/account/verification/phone') * We act like we're updating and validating the verification token but actually we don't need it anymore. */ $dbForProject->deleteDocument('tokens', $verification); - $dbForProject->deleteCachedDocument('users', $profile->getId()); + $dbForProject->purgeCachedDocument('users', $profile->getId()); $queueForEvents ->setParam('userId', $user->getId()) diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index e0d967eb00..19e0f3cf92 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -116,7 +116,7 @@ $getUserGitHub = function (string $userId, Document $project, Database $dbForPro Authorization::skip(fn () => $dbForProject->updateDocument('sessions', $gitHubSession->getId(), $gitHubSession)); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); } catch (Throwable $err) { $index = 0; do { diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index f47e3f8265..e9120e3b6c 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -153,13 +153,13 @@ function createAttribute(string $databaseId, string $collectionId, Document $att } catch (LimitException) { throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded'); } catch (\Exception $e) { - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collectionId); - $dbForProject->deleteCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId); + $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); throw $e; } - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collectionId); - $dbForProject->deleteCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId); + $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) { $twoWayKey = $options['twoWayKey']; @@ -197,13 +197,13 @@ function createAttribute(string $databaseId, string $collectionId, Document $att $dbForProject->deleteDocument('attributes', $attribute->getId()); throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded'); } catch (\Exception $e) { - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId()); + $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); throw $e; } - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId()); + $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); } $queueForDatabase @@ -358,7 +358,7 @@ function updateAttribute( $relatedOptions = \array_merge($relatedAttribute->getAttribute('options'), $options); $relatedAttribute->setAttribute('options', $relatedOptions); $dbForProject->updateDocument('attributes', $db->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $primaryDocumentOptions['twoWayKey'], $relatedAttribute); - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId()); } } else { $dbForProject->updateAttribute( @@ -371,7 +371,7 @@ function updateAttribute( } $attribute = $dbForProject->updateDocument('attributes', $db->getInternalId() . '_' . $collection->getInternalId() . '_' . $key, $attribute); - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collection->getId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collection->getId()); $queueForEvents ->setContext('collection', $collection) @@ -708,8 +708,8 @@ App::delete('/v1/databases/:databaseId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove collection from DB'); } - $dbForProject->deleteCachedDocument('databases', $database->getId()); - $dbForProject->deleteCachedCollection('databases_' . $database->getInternalId()); + $dbForProject->purgeCachedDocument('databases', $database->getId()); + $dbForProject->purgeCachedCollection('databases_' . $database->getInternalId()); $queueForDatabase ->setType(DATABASE_TYPE_DELETE_DATABASE) @@ -1093,7 +1093,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove collection from DB'); } - $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); $queueForDatabase ->setType(DATABASE_TYPE_DELETE_COLLECTION) @@ -2321,8 +2321,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key $attribute = $dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'deleting')); } - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collectionId); - $dbForProject->deleteCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId); + $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) { $options = $attribute->getAttribute('options'); @@ -2343,8 +2343,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'deleting')); } - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $options['relatedCollection']); - $dbForProject->deleteCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $options['relatedCollection']); + $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); } } @@ -2528,7 +2528,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::INDEX_ALREADY_EXISTS); } - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collectionId); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId); $queueForDatabase ->setType(DATABASE_TYPE_CREATE_INDEX) @@ -2701,7 +2701,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') $index = $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'deleting')); } - $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collectionId); + $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId); $queueForDatabase ->setType(DATABASE_TYPE_DELETE_INDEX) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 8461fb2777..72af60b44a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -941,7 +941,7 @@ App::post('/v1/projects/:projectId/webhooks') $webhook = $dbForConsole->createDocument('webhooks', $webhook); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1062,7 +1062,7 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') ->setAttribute('httpPass', $httpPass); $dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response->dynamic($webhook, Response::MODEL_WEBHOOK); }); @@ -1101,7 +1101,7 @@ App::patch('/v1/projects/:projectId/webhooks/:webhookId/signature') $webhook->setAttribute('signatureKey', \bin2hex(\random_bytes(64))); $dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response->dynamic($webhook, Response::MODEL_WEBHOOK); }); @@ -1138,7 +1138,7 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') $dbForConsole->deleteDocument('webhooks', $webhook->getId()); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -1188,7 +1188,7 @@ App::post('/v1/projects/:projectId/keys') $key = $dbForConsole->createDocument('keys', $key); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1302,7 +1302,7 @@ App::put('/v1/projects/:projectId/keys/:keyId') $dbForConsole->updateDocument('keys', $key->getId(), $key); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response->dynamic($key, Response::MODEL_KEY); }); @@ -1339,7 +1339,7 @@ App::delete('/v1/projects/:projectId/keys/:keyId') $dbForConsole->deleteDocument('keys', $key->getId()); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -1389,7 +1389,7 @@ App::post('/v1/projects/:projectId/platforms') $platform = $dbForConsole->createDocument('platforms', $platform); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1504,7 +1504,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') $dbForConsole->updateDocument('platforms', $platform->getId(), $platform); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response->dynamic($platform, Response::MODEL_PLATFORM); }); @@ -1541,7 +1541,7 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') $dbForConsole->deleteDocument('platforms', $platformId); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); $response->noContent(); }); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 2ee351f469..745e5ccbd2 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -114,7 +114,7 @@ App::post('/v1/teams') ]); $membership = $dbForProject->createDocument('memberships', $membership); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); } $queueForEvents->setParam('teamId', $team->getId()); @@ -536,7 +536,7 @@ App::post('/v1/teams/:teamId/memberships') $team->setAttribute('total', $team->getAttribute('total', 0) + 1); $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team)); - $dbForProject->deleteCachedDocument('users', $invitee->getId()); + $dbForProject->purgeCachedDocument('users', $invitee->getId()); } else { try { $membership = $dbForProject->createDocument('memberships', $membership); @@ -843,7 +843,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') /** * Replace membership on profile */ - $dbForProject->deleteCachedDocument('users', $profile->getId()); + $dbForProject->purgeCachedDocument('users', $profile->getId()); $queueForEvents ->setParam('teamId', $team->getId()) @@ -958,13 +958,13 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') Permission::delete(Role::user($user->getId())), ])); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); Authorization::setRole(Role::user($userId)->toString()); $membership = $dbForProject->updateDocument('memberships', $membership->getId(), $membership); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team->setAttribute('total', $team->getAttribute('total', 0) + 1))); @@ -1043,7 +1043,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove membership from DB'); } - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); if ($membership->getAttribute('confirm')) { // Count only confirmed members $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 0869453cc9..3853f08753 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1114,7 +1114,7 @@ App::delete('/v1/users/:userId/sessions/:sessionId') } $dbForProject->deleteDocument('sessions', $session->getId()); - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $queueForEvents ->setParam('userId', $user->getId()) @@ -1158,7 +1158,7 @@ App::delete('/v1/users/:userId/sessions') //TODO: fix this } - $dbForProject->deleteCachedDocument('users', $user->getId()); + $dbForProject->purgeCachedDocument('users', $user->getId()); $queueForEvents ->setParam('userId', $user->getId()) diff --git a/app/controllers/general.php b/app/controllers/general.php index cf383b6710..8bf01f1123 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -232,7 +232,9 @@ App::init() Request::setRoute($route); if ($route === null) { - return $response->setStatusCode(404)->send('Not Found'); + return $response + ->setStatusCode(404) + ->send('Not Found'); } $requestFormat = $request->getHeader('x-appwrite-response-format', App::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', '')); @@ -511,7 +513,7 @@ App::init() if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCCESS)) > $accessedAt) { $key->setAttribute('accessedAt', DateTime::now()); $dbForConsole->updateDocument('keys', $key->getId(), $key); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); } $sdkValidator = new WhiteList($servers, true); @@ -525,7 +527,7 @@ App::init() /** Update access time as well */ $key->setAttribute('accessedAt', Datetime::now()); $dbForConsole->updateDocument('keys', $key->getId(), $key); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); } } } @@ -678,6 +680,7 @@ App::error() Console::error('[Error] Message: ' . $message); Console::error('[Error] File: ' . $file); Console::error('[Error] Line: ' . $line); + Console::error('[Error] Trace: ' . $error->getTraceAsString()); } /** Handle Utopia Errors */ diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index b37d76a816..e5e2ab847b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -350,7 +350,7 @@ App::shutdown() $session = array_shift($sessions); $dbForProject->deleteDocument('sessions', $session->getId()); } - $dbForProject->deleteCachedDocument('users', $userId); + $dbForProject->purgeCachedDocument('users', $userId); }); App::shutdown() diff --git a/app/init.php b/app/init.php index a8e205f1ee..a4801ff707 100644 --- a/app/init.php +++ b/app/init.php @@ -935,22 +935,25 @@ App::setResource('clients', function ($request, $console, $project) { fn ($node) => $node['hostname'], \array_filter( $console->getAttribute('platforms', []), - fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && isset($node['hostname']) && !empty($node['hostname'])) + fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && !empty($node['hostname'])) ) ); - $clients = \array_unique( - \array_merge( - $clientsConsole, - \array_map( - fn ($node) => $node['hostname'], - \array_filter( - $project->getAttribute('platforms', []), - fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB || $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && isset($node['hostname']) && !empty($node['hostname'])) - ) - ) - ) - ); + $clients = $clientsConsole; + $platforms = $project->getAttribute('platforms', []); + + foreach ($platforms as $node) { + if ( + isset($node['type']) && + ($node['type'] === Origin::CLIENT_TYPE_WEB || + $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && + !empty($node['hostname']) + ) { + $clients[] = $node['hostname']; + } + } + + $clients = array_unique($clients); return $clients; }, ['request', 'console', 'project']); diff --git a/composer.lock b/composer.lock index 3f9bf551d2..191286b143 100644 --- a/composer.lock +++ b/composer.lock @@ -1910,12 +1910,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "b7bcc9b373242a9e494b0b7bbf78dfa54333727c" + "reference": "22f9009b06b9d9aa32ca71c76500eaa8fa3366b4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/b7bcc9b373242a9e494b0b7bbf78dfa54333727c", - "reference": "b7bcc9b373242a9e494b0b7bbf78dfa54333727c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/22f9009b06b9d9aa32ca71c76500eaa8fa3366b4", + "reference": "22f9009b06b9d9aa32ca71c76500eaa8fa3366b4", "shasum": "" }, "require": { @@ -1958,7 +1958,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/feat-isolation-mode" }, - "time": "2023-11-22T07:58:21+00:00" + "time": "2023-11-27T01:55:11+00:00" }, { "name": "utopia-php/domains", diff --git a/src/Appwrite/Migration/Version/V15.php b/src/Appwrite/Migration/Version/V15.php index 9d8af4482a..3104341139 100644 --- a/src/Appwrite/Migration/Version/V15.php +++ b/src/Appwrite/Migration/Version/V15.php @@ -372,7 +372,7 @@ class V15 extends Migration } } - $this->projectDB->deleteCachedCollection($table); + $this->projectDB->purgeCachedCollection($table); } /** @@ -479,7 +479,7 @@ class V15 extends Migration $this->createCollection('cache'); Console::log('Created new Collection "variables" collection'); $this->createCollection('variables'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'abuse': diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index c0e2498e7f..4bdae76621 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -48,7 +48,7 @@ class V17 extends Migration try { $this->projectDB->updateAttribute($id, 'mimeType', Database::VAR_STRING, 255, true, false); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'mimeType' from {$id}: {$th->getMessage()}"); } @@ -76,7 +76,7 @@ class V17 extends Migration * Create 'size' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'size'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'size' from {$id}: {$th->getMessage()}"); } @@ -88,7 +88,7 @@ class V17 extends Migration * Update 'mimeType' attribute size (127->255) */ $this->projectDB->updateAttribute($id, 'mimeType', Database::VAR_STRING, 255, true, false); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'mimeType' from {$id}: {$th->getMessage()}"); } @@ -98,7 +98,7 @@ class V17 extends Migration * Create 'bucketInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); } @@ -110,7 +110,7 @@ class V17 extends Migration * Delete 'endTime' attribute (use startTime+duration if needed) */ $this->projectDB->deleteAttribute($id, 'endTime'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'endTime' from {$id}: {$th->getMessage()}"); } @@ -120,7 +120,7 @@ class V17 extends Migration * Rename 'outputPath' to 'path' */ $this->projectDB->renameAttribute($id, 'outputPath', 'path'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'path' from {$id}: {$th->getMessage()}"); } @@ -130,7 +130,7 @@ class V17 extends Migration * Create 'deploymentInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); } @@ -142,7 +142,7 @@ class V17 extends Migration * Delete 'type' attribute */ $this->projectDB->deleteAttribute($id, 'type'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'type' from {$id}: {$th->getMessage()}"); } @@ -154,7 +154,7 @@ class V17 extends Migration * Create 'resourceInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); } @@ -166,7 +166,7 @@ class V17 extends Migration * Create 'deploymentInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); } @@ -176,7 +176,7 @@ class V17 extends Migration * Create 'scheduleInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'scheduleInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'scheduleInternalId' from {$id}: {$th->getMessage()}"); } @@ -186,7 +186,7 @@ class V17 extends Migration * Delete 'scheduleUpdatedAt' attribute */ $this->projectDB->deleteAttribute($id, 'scheduleUpdatedAt'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'scheduleUpdatedAt' from {$id}: {$th->getMessage()}"); } @@ -198,7 +198,7 @@ class V17 extends Migration * Create 'resourceInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}"); } @@ -208,7 +208,7 @@ class V17 extends Migration * Create 'buildInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'buildInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'buildInternalId' from {$id}: {$th->getMessage()}"); } @@ -220,7 +220,7 @@ class V17 extends Migration * Create 'functionInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}"); } @@ -230,7 +230,7 @@ class V17 extends Migration * Create 'deploymentInternalId' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}"); } diff --git a/src/Appwrite/Migration/Version/V18.php b/src/Appwrite/Migration/Version/V18.php index 12d573e549..ac4093aaca 100644 --- a/src/Appwrite/Migration/Version/V18.php +++ b/src/Appwrite/Migration/Version/V18.php @@ -106,7 +106,7 @@ class V18 extends Migration * Create 'passwordHistory' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'passwordHistory'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'passwordHistory' from {$id}: {$th->getMessage()}"); } @@ -117,7 +117,7 @@ class V18 extends Migration * Create 'prefs' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'prefs'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'prefs' from {$id}: {$th->getMessage()}"); } @@ -128,7 +128,7 @@ class V18 extends Migration * Create 'options' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'options'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'options' from {$id}: {$th->getMessage()}"); } diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index c04a3b03de..8083e36426 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -100,7 +100,7 @@ class V19 extends Migration try { $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files'); - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); } catch (\Throwable $th) { Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}"); } @@ -160,7 +160,7 @@ class V19 extends Migration Console::warning("'error' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'buckets': @@ -188,7 +188,7 @@ class V19 extends Migration } } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'builds': @@ -211,7 +211,7 @@ class V19 extends Migration Console::warning("'path' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'certificates': @@ -227,7 +227,7 @@ class V19 extends Migration Console::warning("'logs' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'databases': @@ -237,7 +237,7 @@ class V19 extends Migration Console::warning("'enabled' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'deployments': @@ -300,7 +300,7 @@ class V19 extends Migration } } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'executions': @@ -361,7 +361,7 @@ class V19 extends Migration Console::warning("'_key_responseStatusCode' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'files': @@ -388,7 +388,7 @@ class V19 extends Migration } } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'functions': @@ -450,7 +450,7 @@ class V19 extends Migration } } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'memberships': @@ -460,7 +460,7 @@ class V19 extends Migration Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); // Intentional fall through to update memberships.userInternalId case 'sessions': @@ -471,7 +471,7 @@ class V19 extends Migration Console::warning("'userInternalId' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'domains': @@ -484,7 +484,7 @@ class V19 extends Migration Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'projects': @@ -502,7 +502,7 @@ class V19 extends Migration } } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'stats': @@ -515,26 +515,26 @@ class V19 extends Migration // Holding off on these until a future release // try { // $this->projectDB->deleteAttribute($id, 'type'); - // $this->projectDB->deleteCachedCollection($id); + // $this->projectDB->purgeCachedCollection($id); // } catch (\Throwable $th) { // Console::warning("'type' from {$id}: {$th->getMessage()}"); // } // try { // $this->projectDB->deleteIndex($id, '_key_metric_period_time'); - // $this->projectDB->deleteCachedCollection($id); + // $this->projectDB->purgeCachedCollection($id); // } catch (\Throwable $th) { // Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); // } // try { // $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time'); - // $this->projectDB->deleteCachedCollection($id); + // $this->projectDB->purgeCachedCollection($id); // } catch (\Throwable $th) { // Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}"); // } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'users': @@ -562,7 +562,7 @@ class V19 extends Migration Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; case 'variables': @@ -610,7 +610,7 @@ class V19 extends Migration } } - $this->projectDB->deleteCachedCollection($id); + $this->projectDB->purgeCachedCollection($id); break; default: @@ -753,7 +753,7 @@ class V19 extends Migration Console::warning("'domains' from projects: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection('projects'); + $this->projectDB->purgeCachedCollection('projects'); try { $this->projectDB->deleteAttribute('builds', 'stderr'); @@ -767,7 +767,7 @@ class V19 extends Migration Console::warning("'stdout' from builds: {$th->getMessage()}"); } - $this->projectDB->deleteCachedCollection('builds'); + $this->projectDB->purgeCachedCollection('builds'); } /** diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 5a04df04ba..860cc3a8a2 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -113,7 +113,7 @@ class DeleteOrphanedProjects extends Action foreach ($collections as $collection) { if ($commit) { $dbForProject->deleteCollection($collection->getId()); - $dbForConsole->deleteCachedCollection($collection->getId()); + $dbForConsole->purgeCachedCollection($collection->getId()); } Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); } @@ -121,12 +121,12 @@ class DeleteOrphanedProjects extends Action if ($commit) { $dbForConsole->deleteDocument('projects', $project->getId()); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); if ($dbForProject->exists($dbForProject->getDatabase(), Database::METADATA)) { try { $dbForProject->deleteCollection(Database::METADATA); - $dbForProject->deleteCachedCollection(Database::METADATA); + $dbForProject->purgeCachedCollection(Database::METADATA); } catch (\Throwable $th) { Console::warning('Metadata collection does not exist'); } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index e0ec75e1d4..a90dd742e3 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -186,10 +186,10 @@ class Databases extends Action } if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) { - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); + $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); } - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); + $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } /** @@ -336,12 +336,12 @@ class Databases extends Action } } - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); - $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); + $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) { - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); + $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); } } @@ -402,7 +402,7 @@ class Databases extends Action $this->trigger($database, $collection, $index, $project, $projectId, $events); } - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); + $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } /** @@ -459,7 +459,7 @@ class Databases extends Action $this->trigger($database, $collection, $index, $project, $projectId, $events); } - $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId()); + $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } /** @@ -514,8 +514,8 @@ class Databases extends Action } $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); - $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); + $dbForProject->purgeCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); } $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b95a13a12e..ad3621c46e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -311,8 +311,8 @@ class Deletes extends Action } $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); - $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); - $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); + $dbForProject->purgeCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); + $dbForProject->purgeCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); } $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); @@ -370,7 +370,7 @@ class Deletes extends Action $dbForProject, function (Document $membership) use ($dbForProject) { $userId = $membership->getAttribute('userId'); - $dbForProject->deleteCachedDocument('users', $userId); + $dbForProject->purgeCachedDocument('users', $userId); } ); } @@ -504,7 +504,7 @@ class Deletes extends Action Query::equal('userInternalId', [$userInternalId]) ], $dbForProject); - $dbForProject->deleteCachedDocument('users', $userId); + $dbForProject->purgeCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 31b0df59a3..0bbb3acc5f 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -220,7 +220,7 @@ class Migrations extends Action ]); $this->dbForConsole->createDocument('keys', $key); - $this->dbForConsole->deleteCachedDocument('projects', $project->getId()); + $this->dbForConsole->purgeCachedDocument('projects', $project->getId()); return $key; } From b9ac8e644dcc797e948e6597c0349298d8fece21 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:26:35 +1300 Subject: [PATCH 014/406] Set/reset tenant on init --- app/init.php | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/app/init.php b/app/init.php index a4801ff707..85c618e63c 100644 --- a/app/init.php +++ b/app/init.php @@ -953,9 +953,7 @@ App::setResource('clients', function ($request, $console, $project) { } } - $clients = array_unique($clients); - - return $clients; + return \array_unique($clients); }, ['request', 'console', 'project']); App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForConsole) { @@ -1127,22 +1125,26 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, $database = new Database($dbAdapter, $cache); $database - ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); if ($project->getAttribute('shareTables')) { $database - ->setNamespace('') ->setShareTables(true) - ->setTenant($project->getId()); + ->setTenant($project->getInternalId()) + ->setNamespace(''); + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); } return $database; }, ['pools', 'dbForConsole', 'cache', 'project']); -App::setResource('dbForConsole', function (Group $pools, Cache $cache, Document $project) { +App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $dbAdapter = $pools ->get('console') ->pop() @@ -1151,14 +1153,13 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache, Document $database = new Database($dbAdapter, $cache); $database - ->setTenant($project->getId()) ->setNamespace('_console') ->setMetadata('host', \gethostname()) ->setMetadata('project', 'console') ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); return $database; -}, ['pools', 'cache', 'project']); +}, ['pools', 'cache']); App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools @@ -1172,16 +1173,20 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $configure = (function (Database $database) use ($project) { $database - ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); if ($project->getAttribute('shareTables')) { $database - ->setNamespace('') ->setShareTables(true) - ->setTenant($project->getId()); + ->setTenant($project->getInternalId()) + ->setNamespace(''); + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); } }); From 68548ebd478d2f8ef559c9d2845bf998555e5556 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:27:16 +1300 Subject: [PATCH 015/406] Fix worker tenant set/reset --- app/worker.php | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/app/worker.php b/app/worker.php index 98dc9e682e..b3382193f9 100644 --- a/app/worker.php +++ b/app/worker.php @@ -72,9 +72,12 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, if ($project->getAttribute('shareTables')) { $database ->setShareTables(true) - ->setTenant($project->getId()); + ->setTenant($project->getInternalId()) + ->setNamespace(''); } else { $database + ->setShareTables(false) + ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } @@ -94,14 +97,19 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; + if ($project->getAttribute('shareTables')) { $database ->setShareTables(true) - ->setTenant($project->getId()); + ->setTenant($project->getInternalId()) + ->setNamespace(''); } else { $database + ->setShareTables(false) + ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } + return $database; } @@ -117,9 +125,12 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if ($project->getAttribute('shareTables')) { $database ->setShareTables(true) - ->setTenant($project->getId()); + ->setTenant($project->getInternalId()) + ->setNamespace(''); } else { $database + ->setShareTables(false) + ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } @@ -228,7 +239,7 @@ try { * Any worker can be configured with the following env vars: * - _APP_WORKERS_NUM The total number of worker processes * - _APP_WORKER_PER_CORE The number of worker processes per core (ignored if _APP_WORKERS_NUM is set) - * - _APP_QUEUE_NAME The name of the queue to read for database events + * - _APP_QUEUE_NAME The name of the queue to read for database events */ if ($workerName === 'databases') { $queueName = App::getEnv('_APP_QUEUE_NAME', 'database_db_main'); From 9472bff8f9f3211c58ddfd345a53a1ee0c3bc7b2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:27:41 +1300 Subject: [PATCH 016/406] Fix project create tenant set/reset --- app/controllers/api/projects.php | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 72af60b44a..994f842184 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -9,6 +9,7 @@ use Appwrite\Network\Validator\Origin; use Appwrite\Template\Template; use Appwrite\Utopia\Database\Validator\ProjectId; use Appwrite\Utopia\Database\Validator\Queries\Projects; +use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use PHPMailer\PHPMailer\PHPMailer; use Utopia\Abuse\Adapters\TimeLimit; @@ -71,11 +72,12 @@ App::post('/v1/projects') ->param('legalCity', '', new Text(256), 'Project legal City. Max length: 256 chars.', true) ->param('legalAddress', '', new Text(256), 'Project legal Address. Max length: 256 chars.', true) ->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true) + ->inject('request') ->inject('response') ->inject('dbForConsole') ->inject('cache') ->inject('pools') - ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { + ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Request $request, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { $team = $dbForConsole->getDocument('teams', $teamId); @@ -98,8 +100,6 @@ App::post('/v1/projects') $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId; - $dbForConsole->setTenant($projectId); - $backups['database_db_fra1_v14x_02'] = ['from' => '03:00', 'to' => '05:00']; $backups['database_db_fra1_v14x_03'] = ['from' => '00:00', 'to' => '02:00']; $backups['database_db_fra1_v14x_04'] = ['from' => '00:00', 'to' => '02:00']; @@ -141,8 +141,13 @@ App::post('/v1/projects') throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); } - static $counter = 0; - $shareTables = $counter++ % 2 === 0; + // One in 20 projects use shared tables + $shareTables = !\mt_rand(0, 19); + + // Allow overriding in development mode + if (App::isDevelopment()) { + $shareTables = (bool) $request->getHeader('x-appwrite-share-tables', false); + } try { $project = $dbForConsole->createDocument('projects', new Document([ @@ -187,10 +192,13 @@ App::post('/v1/projects') if ($shareTables) { $dbForProject ->setShareTables(true) - ->setTenant($project->getId()); + ->setTenant($project->getInternalId()) + ->setNamespace(''); } else { $dbForProject - ->setNamespace("_{$project->getInternalId()}"); + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); } $dbForProject->create(); From 0885f31dd72e8e466f6cd7f774514b4e6c3cfc3b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:29:23 +1300 Subject: [PATCH 017/406] Update database --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 191286b143..858fe2284c 100644 --- a/composer.lock +++ b/composer.lock @@ -1910,12 +1910,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "22f9009b06b9d9aa32ca71c76500eaa8fa3366b4" + "reference": "8756141bef49993643acea5120314b9390bfe30a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/22f9009b06b9d9aa32ca71c76500eaa8fa3366b4", - "reference": "22f9009b06b9d9aa32ca71c76500eaa8fa3366b4", + "url": "https://api.github.com/repos/utopia-php/database/zipball/8756141bef49993643acea5120314b9390bfe30a", + "reference": "8756141bef49993643acea5120314b9390bfe30a", "shasum": "" }, "require": { @@ -1958,7 +1958,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/feat-isolation-mode" }, - "time": "2023-11-27T01:55:11+00:00" + "time": "2023-11-27T07:58:37+00:00" }, { "name": "utopia-php/domains", From 82dfe5ddee4f82f47cb08fd450a859de5ed0208f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:30:02 +1300 Subject: [PATCH 018/406] Add tests --- app/controllers/general.php | 4 +- .../Projects/ProjectsConsoleClientTest.php | 470 ++++++++++++++++++ .../Projects/ProjectsCustomClientTest.php | 5 - .../Projects/ProjectsCustomServerTest.php | 5 - 4 files changed, 472 insertions(+), 12 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 8bf01f1123..eac637eeb3 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -427,7 +427,7 @@ App::init() ->addHeader('Server', 'Appwrite') ->addHeader('X-Content-Type-Options', 'nosniff') ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE') - ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma') + ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Share-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma') ->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies') ->addHeader('Access-Control-Allow-Origin', $refDomain) ->addHeader('Access-Control-Allow-Credentials', 'true'); @@ -591,7 +591,7 @@ App::options() $response ->addHeader('Server', 'Appwrite') ->addHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE') - ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies') + ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Share-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies') ->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies') ->addHeader('Access-Control-Allow-Origin', $origin) ->addHeader('Access-Control-Allow-Credentials', 'true') diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index f90a04f290..b0172b9d2e 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -8,6 +8,7 @@ use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\SideClient; use Tests\E2E\Client; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Helpers\ID; @@ -3294,4 +3295,473 @@ class ProjectsConsoleClientTest extends Scope return $data; } + + public function testTenantIsolation(): void + { + // Create a team and a project + $team = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'teamId' => ID::unique(), + 'name' => 'Amazing Team', + ]); + + $teamId = $team['body']['$id']; + + // Project-level isolation + $project1 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-share-tables' => false + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Application level isolation (shared tables) + $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-share-tables' => true + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Project-level isolation + $project3 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-share-tables' => false + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Application level isolation (shared tables) + $project4 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-share-tables' => true + ], $this->getHeaders()), [ + 'projectId' => ID::unique(), + 'name' => 'Amazing Project', + 'teamId' => $teamId, + 'region' => 'default' + ]); + + // Create and API key in each project + $key1 = $this->client->call(Client::METHOD_POST, '/projects/' . $project1['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write'], + ]); + + $key2 = $this->client->call(Client::METHOD_POST, '/projects/' . $project2['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write'], + ]); + + $key3 = $this->client->call(Client::METHOD_POST, '/projects/' . $project3['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write'], + ]); + + $key4 = $this->client->call(Client::METHOD_POST, '/projects/' . $project4['body']['$id'] . '/keys', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Key Test', + 'scopes' => ['databases.read', 'databases.write', 'collections.read', 'collections.write', 'attributes.read', 'attributes.write', 'indexes.read', 'indexes.write', 'documents.read', 'documents.write'], + ]); + + // Create a database in each project + $database1 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + $database2 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + $database3 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + $database4 = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Amazing Database', + ]); + + // Create a collection in each project + $collection1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => $database1['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + $collection2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + $collection3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => $database3['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + $collection4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => $database4['body']['$id'], + 'collectionId' => ID::unique(), + 'name' => 'Amazing Collection', + ]); + + // Create an attribute in each project + $attribute1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => $database1['body']['$id'], + 'collectionId' => $collection1['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + $attribute2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => $collection2['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + $attribute3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => $database3['body']['$id'], + 'collectionId' => $collection3['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + $attribute4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => $database4['body']['$id'], + 'collectionId' => $collection4['body']['$id'], + 'key' => ID::unique(), + 'size' => 255, + 'required' => true + ]); + + // Wait for attributes + \sleep(2); + + // Create an index in each project + $index1 = $this->client->call(Client::METHOD_POST, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ], [ + 'databaseId' => $database1['body']['$id'], + 'collectionId' => $collection1['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute1['body']['key']], + ]); + + $index2 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => $collection2['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute2['body']['key']], + ]); + + $index3 = $this->client->call(Client::METHOD_POST, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ], [ + 'databaseId' => $database3['body']['$id'], + 'collectionId' => $collection3['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute3['body']['key']], + ]); + + $index4 = $this->client->call(Client::METHOD_POST, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ], [ + 'databaseId' => $database4['body']['$id'], + 'collectionId' => $collection4['body']['$id'], + 'key' => ID::unique(), + 'type' => Database::INDEX_KEY, + 'attributes' => [$attribute4['body']['key']], + ]); + + // Wait for indexes + \sleep(2); + + // Assert that each project has only 1 database, 1 collection, 1 attribute and 1 index + $databasesProject1 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject1['body']['total']); + $this->assertEquals(1, \count($databasesProject1['body']['databases'])); + + $databasesProject2 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject2['body']['total']); + $this->assertEquals(1, \count($databasesProject2['body']['databases'])); + + $databasesProject3 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject3['body']['total']); + $this->assertEquals(1, \count($databasesProject3['body']['databases'])); + + $databasesProject4 = $this->client->call(Client::METHOD_GET, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $databasesProject4['body']['total']); + $this->assertEquals(1, \count($databasesProject4['body']['databases'])); + + $collectionsProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject1['body']['total']); + $this->assertEquals(1, \count($collectionsProject1['body']['collections'])); + + $collectionsProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject2['body']['total']); + $this->assertEquals(1, \count($collectionsProject2['body']['collections'])); + + $collectionsProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject3['body']['total']); + $this->assertEquals(1, \count($collectionsProject3['body']['collections'])); + + $collectionsProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $collectionsProject4['body']['total']); + $this->assertEquals(1, \count($collectionsProject4['body']['collections'])); + + $attributesProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject1['body']['total']); + $this->assertEquals(1, \count($attributesProject1['body']['attributes'])); + + $attributesProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject2['body']['total']); + $this->assertEquals(1, \count($attributesProject2['body']['attributes'])); + + $attributesProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject3['body']['total']); + $this->assertEquals(1, \count($attributesProject3['body']['attributes'])); + + $attributesProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/attributes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $attributesProject4['body']['total']); + $this->assertEquals(1, \count($attributesProject4['body']['attributes'])); + + $indexesProject1 = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject1['body']['total']); + $this->assertEquals(1, \count($indexesProject1['body']['indexes'])); + + $indexesProject2 = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject2['body']['total']); + $this->assertEquals(1, \count($indexesProject2['body']['indexes'])); + + $indexesProject3 = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject3['body']['total']); + $this->assertEquals(1, \count($indexesProject3['body']['indexes'])); + + $indexesProject4 = $this->client->call(Client::METHOD_GET, '/databases/' . $database4['body']['$id'] . '/collections/' . $collection4['body']['$id'] . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project4['body']['$id'], + 'x-appwrite-key' => $key4['body']['secret'] + ]); + + $this->assertEquals(1, $indexesProject4['body']['total']); + $this->assertEquals(1, \count($indexesProject4['body']['indexes'])); + + // Attempt to read cross-type resources + $collectionProject2WithProject1Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database2['body']['$id'] . '/collections/' . $collection2['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject2WithProject1Key['headers']['status-code']); + + $collectionProject1WithProject2Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject1WithProject2Key['headers']['status-code']); + + // Attempt to read cross-tenant resources + $collectionProject3WithProject1Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database3['body']['$id'] . '/collections/' . $collection3['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project1['body']['$id'], + 'x-appwrite-key' => $key1['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject3WithProject1Key['headers']['status-code']); + + $collectionProject1WithProject3Key = $this->client->call(Client::METHOD_GET, '/databases/' . $database1['body']['$id'] . '/collections/' . $collection1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project3['body']['$id'], + 'x-appwrite-key' => $key3['body']['secret'] + ]); + + $this->assertEquals(404, $collectionProject1WithProject3Key['headers']['status-code']); + + // Assert that shared project resources can have the same ID as they're unique on tenant + ID not just ID + $collection5 = $this->client->call(Client::METHOD_POST, '/databases/' . $database2['body']['$id'] . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project2['body']['$id'], + 'x-appwrite-key' => $key2['body']['secret'] + ], [ + 'databaseId' => $database2['body']['$id'], + 'collectionId' => $collection4['body']['$id'], + 'name' => 'Amazing Collection', + ]); + + $this->assertEquals(201, $collection5['headers']['status-code']); + } } diff --git a/tests/e2e/Services/Projects/ProjectsCustomClientTest.php b/tests/e2e/Services/Projects/ProjectsCustomClientTest.php index 9dfd48ce50..ddef681e61 100644 --- a/tests/e2e/Services/Projects/ProjectsCustomClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsCustomClientTest.php @@ -11,9 +11,4 @@ class ProjectsCustomClientTest extends Scope { use ProjectCustom; use SideClient; - - public function testMock() - { - $this->assertEquals(true, true); - } } diff --git a/tests/e2e/Services/Projects/ProjectsCustomServerTest.php b/tests/e2e/Services/Projects/ProjectsCustomServerTest.php index 1c5b48f146..2d11896676 100644 --- a/tests/e2e/Services/Projects/ProjectsCustomServerTest.php +++ b/tests/e2e/Services/Projects/ProjectsCustomServerTest.php @@ -11,9 +11,4 @@ class ProjectsCustomServerTest extends Scope { use ProjectCustom; use SideServer; - - public function testMock() - { - $this->assertEquals(true, true); - } } From edee5400ba873df0bfefa3ceead352d837118171 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:43:21 +1300 Subject: [PATCH 019/406] Fix casing --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 85c618e63c..ff797b5fbc 100644 --- a/app/init.php +++ b/app/init.php @@ -1264,7 +1264,7 @@ function getDevice($root): Device return match ($device) { Storage::DEVICE_S3 => new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl), - STORAGE::DEVICE_DO_SPACES => new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl), + Storage::DEVICE_DO_SPACES => new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl), Storage::DEVICE_BACKBLAZE => new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl), Storage::DEVICE_LINODE => new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl), Storage::DEVICE_WASABI => new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl), From 2e89113e6a4ff1f6bb9a2d05f3b00da15540e282 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:52:46 +1300 Subject: [PATCH 020/406] Self review --- .gitignore | 1 + app/worker.php | 1 - appwrite.json | 4 ---- 3 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 appwrite.json diff --git a/.gitignore b/.gitignore index 3151de5adb..15478fcfab 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ debug/ app/sdks dev/yasd_init.php .phpunit.result.cache +appwrite.json diff --git a/app/worker.php b/app/worker.php index b3382193f9..1ef1d340a6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -81,7 +81,6 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setNamespace('_' . $project->getInternalId()); } - return $database; }, ['cache', 'register', 'message', 'dbForConsole']); diff --git a/appwrite.json b/appwrite.json deleted file mode 100644 index e3d948ed1a..0000000000 --- a/appwrite.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "projectId": "console", - "projectName": "" -} \ No newline at end of file From c4806e883edf3a13cbcfabda86e90bea3e5938b5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 01:00:22 +1300 Subject: [PATCH 021/406] Update lock --- composer.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/composer.lock b/composer.lock index a5d1fb327a..c61e497ab6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2c23008af30193734b6717a41632aa54", + "content-hash": "75330561b68809c4c842c513ead64783", "packages": [ { "name": "adhocore/jwt", @@ -1910,12 +1910,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "8756141bef49993643acea5120314b9390bfe30a" + "reference": "ba6d104d2d0ac2de115bb426b01cdcbdc5143ce6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/8756141bef49993643acea5120314b9390bfe30a", - "reference": "8756141bef49993643acea5120314b9390bfe30a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/ba6d104d2d0ac2de115bb426b01cdcbdc5143ce6", + "reference": "ba6d104d2d0ac2de115bb426b01cdcbdc5143ce6", "shasum": "" }, "require": { @@ -1958,7 +1958,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/feat-isolation-mode" }, - "time": "2023-11-27T07:58:37+00:00" + "time": "2023-11-27T11:35:51+00:00" }, { "name": "utopia-php/domains", From c1f10b06a3db68df9e5c424715a597a44e2d3ce3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 01:34:29 +1300 Subject: [PATCH 022/406] Fix index limit test --- composer.lock | 8 ++++---- .../e2e/Services/Databases/DatabasesCustomServerTest.php | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index c61e497ab6..04d20d6fb5 100644 --- a/composer.lock +++ b/composer.lock @@ -1910,12 +1910,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "ba6d104d2d0ac2de115bb426b01cdcbdc5143ce6" + "reference": "c1341e2daa90aaa93511a98838cd48243aa2eae5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/ba6d104d2d0ac2de115bb426b01cdcbdc5143ce6", - "reference": "ba6d104d2d0ac2de115bb426b01cdcbdc5143ce6", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c1341e2daa90aaa93511a98838cd48243aa2eae5", + "reference": "c1341e2daa90aaa93511a98838cd48243aa2eae5", "shasum": "" }, "require": { @@ -1958,7 +1958,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/feat-isolation-mode" }, - "time": "2023-11-27T11:35:51+00:00" + "time": "2023-11-27T12:29:12+00:00" }, { "name": "utopia-php/domains", diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 648a4de800..aa1d91dcdf 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -1374,10 +1374,10 @@ class DatabasesCustomServerTest extends Scope } // Test indexLimit = 64 - // MariaDB, MySQL, and MongoDB create 5 indexes per new collection + // MariaDB, MySQL, and MongoDB create 6 indexes per new collection // Add up to the limit, then check if the next index throws IndexLimitException - for ($i = 0; $i < 59; $i++) { // $this->assertEquals(true, static::getDatabase()->createIndex('indexLimit', "index{$i}", Database::INDEX_KEY, ["test{$i}"], [16])); + for ($i = 0; $i < 58; $i++) { $index = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/indexes', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1405,7 +1405,7 @@ class DatabasesCustomServerTest extends Scope $this->assertIsArray($collection['body']['attributes']); $this->assertIsArray($collection['body']['indexes']); $this->assertCount(64, $collection['body']['attributes']); - $this->assertCount(59, $collection['body']['indexes']); + $this->assertCount(58, $collection['body']['indexes']); $tooMany = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/indexes', array_merge([ 'content-type' => 'application/json', From 632c902e1c9bc88a2c712a34444220968cbbbc0d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 01:53:24 +1300 Subject: [PATCH 023/406] Remove debug trace --- app/controllers/general.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 441976f7fb..1880e3826d 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -685,7 +685,6 @@ App::error() Console::error('[Error] Message: ' . $message); Console::error('[Error] File: ' . $file); Console::error('[Error] Line: ' . $line); - Console::error('[Error] Trace: ' . $error->getTraceAsString()); } /** Handle Utopia Errors */ From 7104f9e2c840e07284a154d816f2c1e5a9125208 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 02:08:04 +1300 Subject: [PATCH 024/406] Add comments marking `shareTables` project attributes + header overrides as temporary --- app/config/collections.php | 1 + app/controllers/api/projects.php | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/config/collections.php b/app/config/collections.php index 60eeaf40a8..ed5e94eb65 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3269,6 +3269,7 @@ $consoleCollections = array_merge([ 'filters' => [], ], [ + // To be removed once all project are using shared tables '$id' => ID::custom('shareTables'), 'type' => Database::VAR_BOOLEAN, 'format' => '', diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 994f842184..330f89dffd 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -144,7 +144,7 @@ App::post('/v1/projects') // One in 20 projects use shared tables $shareTables = !\mt_rand(0, 19); - // Allow overriding in development mode + // Allow overriding in development mode, to be removed once all project are using shared tables if (App::isDevelopment()) { $shareTables = (bool) $request->getHeader('x-appwrite-share-tables', false); } From 4f539f1ed48a99fdec0f27be759d38d6192a1585 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Nov 2023 16:04:15 +1300 Subject: [PATCH 025/406] Update switch to match --- app/init.php | 85 +++++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/app/init.php b/app/init.php index f2063870ac..82e2baf804 100644 --- a/app/init.php +++ b/app/init.php @@ -1290,46 +1290,51 @@ function getDevice($root): Device default => new Local($root), }; } else { - switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) { - case Storage::DEVICE_LOCAL: - default: - return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - case Storage::DEVICE_LINODE: - $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); - case Storage::DEVICE_WASABI: - $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); - } + $device = strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? ''); + + return match ($device) { + Storage::DEVICE_S3 => new S3( + $root, + App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''), + App::getEnv('_APP_STORAGE_S3_SECRET', ''), + App::getEnv('_APP_STORAGE_S3_BUCKET', ''), + App::getEnv('_APP_STORAGE_S3_REGION', ''), + 'private' + ), + Storage::DEVICE_DO_SPACES => new DOSpaces( + $root, + App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''), + App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''), + App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''), + App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''), + 'private' + ), + Storage::DEVICE_BACKBLAZE => new Backblaze( + $root, + App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''), + App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''), + App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''), + App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''), + 'private' + ), + Storage::DEVICE_LINODE => new Linode( + $root, + App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''), + App::getEnv('_APP_STORAGE_LINODE_SECRET', ''), + App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''), + App::getEnv('_APP_STORAGE_LINODE_REGION', ''), + 'private' + ), + Storage::DEVICE_WASABI => new Wasabi( + $root, + App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''), + App::getEnv('_APP_STORAGE_WASABI_SECRET', ''), + App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''), + App::getEnv('_APP_STORAGE_WASABI_REGION', ''), + 'private' + ), + default => new Local($root), + }; } } From e109f1e0080a59b9c91e0533b0ce1ce846319c6e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Nov 2023 16:06:49 +1300 Subject: [PATCH 026/406] Add metadata in worker dbs --- app/worker.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/worker.php b/app/worker.php index 1ef1d340a6..41c5a06a9d 100644 --- a/app/worker.php +++ b/app/worker.php @@ -69,6 +69,10 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, $database = new Database($adapter, $cache); + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()); + if ($project->getAttribute('shareTables')) { $database ->setShareTables(true) From 7e12da5a2aaa81342a42f297dbb93c86023c42d0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Nov 2023 16:32:15 +1300 Subject: [PATCH 027/406] Use hard-coded DB name instead of project attribute --- app/controllers/api/projects.php | 13 +++++++------ app/init.php | 7 ++++--- app/worker.php | 6 +++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 330f89dffd..9916ae67f2 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -129,7 +129,7 @@ App::post('/v1/projects') } } - $databaseOverride = App::getEnv('_APP_DATABASE_OVERRIDE', null); + $databaseOverride = App::getEnv('_APP_DATABASE_OVERRIDE'); $index = array_search($databaseOverride, $databases); if ($index !== false) { $database = $databases[$index]; @@ -142,11 +142,13 @@ App::post('/v1/projects') } // One in 20 projects use shared tables - $shareTables = !\mt_rand(0, 19); + if (!\mt_rand(0, 19)) { + $database = DATABASE_SHARED_TABLES; + } - // Allow overriding in development mode, to be removed once all project are using shared tables - if (App::isDevelopment()) { - $shareTables = (bool) $request->getHeader('x-appwrite-share-tables', false); + // Allow overriding in development mode, to be removed once all project are using shared tables. + if (App::isDevelopment() && $request->getHeader('x-appwrite-share-tables', false)) { + $database = DATABASE_SHARED_TABLES; } try { @@ -181,7 +183,6 @@ App::post('/v1/projects') 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), 'database' => $database, - 'shareTables' => $shareTables, ])); } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); diff --git a/app/init.php b/app/init.php index 82e2baf804..214f91ae71 100644 --- a/app/init.php +++ b/app/init.php @@ -140,7 +140,8 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; const APP_HOSTNAME_INTERNAL = 'appwrite'; -// Database Reconnect +// Database +const DATABASE_SHARED_TABLES = 'database_db_self_hosted_shared_tables'; const DATABASE_RECONNECT_SLEEP = 2; const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; // Database Worker Types @@ -1148,7 +1149,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - if ($project->getAttribute('shareTables')) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -1196,7 +1197,7 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - if ($project->getAttribute('shareTables')) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/app/worker.php b/app/worker.php index 41c5a06a9d..71de280bfb 100644 --- a/app/worker.php +++ b/app/worker.php @@ -73,7 +73,7 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()); - if ($project->getAttribute('shareTables')) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -101,7 +101,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; - if ($project->getAttribute('shareTables')) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -125,7 +125,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $databases[$databaseName] = $database; - if ($project->getAttribute('shareTables')) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) From 6b0586eaf60129d1bb5ef481caf4e26a39f08cc9 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 8 Dec 2023 18:06:53 +0100 Subject: [PATCH 028/406] Remove DB attribute --- app/config/collections.php | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index ed5e94eb65..db229ce87a 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3268,18 +3268,6 @@ $consoleCollections = array_merge([ 'array' => false, 'filters' => [], ], - [ - // To be removed once all project are using shared tables - '$id' => ID::custom('shareTables'), - 'type' => Database::VAR_BOOLEAN, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, From 4245c0b86e19cc812ca4c08da9d66c038c0de25d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 8 Dec 2023 18:07:38 +0100 Subject: [PATCH 029/406] Update dependencies --- composer.json | 6 ++--- composer.lock | 69 +++++++++++++++++---------------------------------- 2 files changed, 26 insertions(+), 49 deletions(-) diff --git a/composer.json b/composer.json index 799f5b4987..3f24d9da74 100644 --- a/composer.json +++ b/composer.json @@ -43,13 +43,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.13.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "dev-feat-isolation-modes as 0.33.0", + "utopia-php/abuse": "0.34.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "dev-feat-isolation-modes as 0.35.0", + "utopia-php/audit": "0.36.*", "utopia-php/cache": "0.8.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-isolation-mode as 0.45.2", + "utopia-php/database": "0.46.*", "utopia-php/domains": "0.3.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.31.0", diff --git a/composer.lock b/composer.lock index 04d20d6fb5..f591332dd0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "75330561b68809c4c842c513ead64783", + "content-hash": "4282a4b17ae483c99bb529c9b8cf7198", "packages": [ { "name": "adhocore/jwt", @@ -1615,23 +1615,23 @@ }, { "name": "utopia-php/abuse", - "version": "dev-feat-isolation-modes", + "version": "0.34.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "171eda04bfc53e5e24bdb36230ae84fe686ac2ee" + "reference": "49f84abc0f2317ba5b55af809cf6b411253c4855" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/171eda04bfc53e5e24bdb36230ae84fe686ac2ee", - "reference": "171eda04bfc53e5e24bdb36230ae84fe686ac2ee", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/49f84abc0f2317ba5b55af809cf6b411253c4855", + "reference": "49f84abc0f2317ba5b55af809cf6b411253c4855", "shasum": "" }, "require": { "ext-curl": "*", "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "dev-feat-isolation-mode as 0.45.0" + "utopia-php/database": "0.46.*" }, "require-dev": { "laravel/pint": "1.5.*", @@ -1658,9 +1658,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/feat-isolation-modes" + "source": "https://github.com/utopia-php/abuse/tree/0.34.0" }, - "time": "2023-11-22T09:49:43+00:00" + "time": "2023-12-08T17:04:15+00:00" }, { "name": "utopia-php/analytics", @@ -1710,21 +1710,21 @@ }, { "name": "utopia-php/audit", - "version": "dev-feat-isolation-modes", + "version": "0.36.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "b561fa872c60e28afda02b4428ad5a2444099c44" + "reference": "b490f06d687fc1510d44ddce305afb81462ddd62" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/b561fa872c60e28afda02b4428ad5a2444099c44", - "reference": "b561fa872c60e28afda02b4428ad5a2444099c44", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/b490f06d687fc1510d44ddce305afb81462ddd62", + "reference": "b490f06d687fc1510d44ddce305afb81462ddd62", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "dev-feat-isolation-mode as 0.45.0" + "utopia-php/database": "0.46.*" }, "require-dev": { "laravel/pint": "1.5.*", @@ -1751,9 +1751,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/feat-isolation-modes" + "source": "https://github.com/utopia-php/audit/tree/0.36.0" }, - "time": "2023-11-22T09:49:57+00:00" + "time": "2023-12-08T17:04:53+00:00" }, { "name": "utopia-php/cache", @@ -1906,16 +1906,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-isolation-mode", + "version": "0.46.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "c1341e2daa90aaa93511a98838cd48243aa2eae5" + "reference": "035e6b4a0005ad87aa73beca8714721925ea8f5e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/c1341e2daa90aaa93511a98838cd48243aa2eae5", - "reference": "c1341e2daa90aaa93511a98838cd48243aa2eae5", + "url": "https://api.github.com/repos/utopia-php/database/zipball/035e6b4a0005ad87aa73beca8714721925ea8f5e", + "reference": "035e6b4a0005ad87aa73beca8714721925ea8f5e", "shasum": "" }, "require": { @@ -1956,9 +1956,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-isolation-mode" + "source": "https://github.com/utopia-php/database/tree/0.46.0" }, - "time": "2023-11-27T12:29:12+00:00" + "time": "2023-12-08T16:02:38+00:00" }, { "name": "utopia-php/domains", @@ -5796,32 +5796,9 @@ "time": "2023-08-28T11:09:02+00:00" } ], - "aliases": [ - { - "package": "utopia-php/abuse", - "version": "dev-feat-isolation-modes", - "alias": "0.33.0", - "alias_normalized": "0.33.0.0" - }, - { - "package": "utopia-php/audit", - "version": "dev-feat-isolation-modes", - "alias": "0.35.0", - "alias_normalized": "0.35.0.0" - }, - { - "package": "utopia-php/database", - "version": "dev-feat-isolation-mode", - "alias": "0.45.2", - "alias_normalized": "0.45.2.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/abuse": 20, - "utopia-php/audit": 20, - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 4710310629326ba2aa7442de926a42ac003ad4e9 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 8 Dec 2023 18:33:35 +0100 Subject: [PATCH 030/406] Check if self-hosted --- .env | 1 + app/controllers/api/projects.php | 17 ++++++--- app/init.php | 60 +++++++++++++------------------- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/.env b/.env index ad551e705a..cbd27c74bb 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ _APP_ENV=development +_APP_EDITION=self-hosted _APP_LOCALE=en _APP_WORKER_PER_CORE=6 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 9916ae67f2..c21de5f83e 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -141,13 +141,20 @@ App::post('/v1/projects') throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); } - // One in 20 projects use shared tables - if (!\mt_rand(0, 19)) { + // TODO: One in 20 projects use shared tables. Temporary until all projects are using shared tables. + if ( + !\mt_rand(0, 19) + && App::getEnv('_APP_EDITION', 'self-hosted') !== 'self-hosted' + ) { $database = DATABASE_SHARED_TABLES; } - // Allow overriding in development mode, to be removed once all project are using shared tables. - if (App::isDevelopment() && $request->getHeader('x-appwrite-share-tables', false)) { + // TODO: Allow overriding in development mode. Temporary until all projects are using shared tables. + if ( + App::isDevelopment() + && App::getEnv('_APP_EDITION', 'self-hosted') !== 'self-hosted' + && $request->getHeader('x-appwrite-share-tables', false) + ) { $database = DATABASE_SHARED_TABLES; } @@ -190,7 +197,7 @@ App::post('/v1/projects') $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); - if ($shareTables) { + if ($database === DATABASE_SHARED_TABLES) { $dbForProject ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/app/init.php b/app/init.php index 515d9f884e..121ef327e2 100644 --- a/app/init.php +++ b/app/init.php @@ -684,47 +684,38 @@ $register->set('pools', function () { /** * Get Resource * - * Creation could be reused accross connection types like database, cache, queue, etc. + * Creation could be reused across connection types like database, cache, queue, etc. * * Resource assignment to an adapter will happen below. */ - switch ($dsnScheme) { - case 'mysql': - case 'mariadb': - $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - )); - }); - }; - break; - case 'redis': - $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); - @$redis->pconnect($dsnHost, (int)$dsnPort); - if ($dsnPass) { - $redis->auth($dsnPass); - } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + $resource = match ($dsnScheme) { + 'mysql', + 'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }, + 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - return $redis; - }; - break; - - default: - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); - break; - } + return $redis; + }, + default => throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"), + }; $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { // Get Adapter - $adapter = null; switch ($type) { case 'database': $adapter = match ($dsn->getScheme()) { @@ -753,7 +744,6 @@ $register->set('pools', function () { default: throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); - break; } return $adapter; From d5428f8f7aa7d32a67fea5cc944dddc2d4d59d9c Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 27 Dec 2023 16:28:53 +0200 Subject: [PATCH 031/406] databases.php collection not found --- app/controllers/api/databases.php | 7 ++- composer.lock | 94 +++++++++++++++---------------- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0d2e2f22d1..8b092e54b1 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -28,6 +28,7 @@ use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Restricted as RestrictedException; use Utopia\Database\Exception\Structure as StructureException; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Timeout as TimeoutException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -2973,7 +2974,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('dbForProject') ->inject('mode') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode) { - $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); + try { + $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); + } catch (DatabaseException $e) { + throw new Exception(Exception::COLLECTION_NOT_FOUND, "Databases Collection not found"); + } $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); diff --git a/composer.lock b/composer.lock index 3518396665..595a63fc99 100644 --- a/composer.lock +++ b/composer.lock @@ -2476,16 +2476,16 @@ }, { "name": "utopia-php/platform", - "version": "0.5.0", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "229a7b1fa1f39afd1532f7a515326a6afc222a26" + "reference": "3eceef0b6593fe0f7d2efd36d40402a395a4c285" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/229a7b1fa1f39afd1532f7a515326a6afc222a26", - "reference": "229a7b1fa1f39afd1532f7a515326a6afc222a26", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/3eceef0b6593fe0f7d2efd36d40402a395a4c285", + "reference": "3eceef0b6593fe0f7d2efd36d40402a395a4c285", "shasum": "" }, "require": { @@ -2493,7 +2493,7 @@ "ext-redis": "*", "php": ">=8.0", "utopia-php/cli": "0.15.*", - "utopia-php/framework": "0.31.*" + "utopia-php/framework": "0.*.*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -2519,9 +2519,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/0.5.0" + "source": "https://github.com/utopia-php/platform/tree/0.5.1" }, - "time": "2023-10-16T20:28:49+00:00" + "time": "2023-12-26T16:14:41+00:00" }, { "name": "utopia-php/pools", @@ -2904,23 +2904,23 @@ }, { "name": "utopia-php/vcs", - "version": "0.6.2", + "version": "0.6.4", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "f135291b87cb45335fc6608722e7f89894bc33ee" + "reference": "b2595a50a4897a8c88319240810055b7a96efd6d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/f135291b87cb45335fc6608722e7f89894bc33ee", - "reference": "f135291b87cb45335fc6608722e7f89894bc33ee", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/b2595a50a4897a8c88319240810055b7a96efd6d", + "reference": "b2595a50a4897a8c88319240810055b7a96efd6d", "shasum": "" }, "require": { "adhocore/jwt": "^1.1", "php": ">=8.0", "utopia-php/cache": "^0.8.0", - "utopia-php/framework": "0.31.*" + "utopia-php/framework": "0.*.*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -2947,9 +2947,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/0.6.2" + "source": "https://github.com/utopia-php/vcs/tree/0.6.4" }, - "time": "2023-11-08T15:36:03+00:00" + "time": "2023-12-26T15:38:19+00:00" }, { "name": "utopia-php/websocket", @@ -3487,16 +3487,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.17.1", + "version": "v4.18.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d" + "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999", + "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999", "shasum": "" }, "require": { @@ -3537,9 +3537,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0" }, - "time": "2023-08-13T19:53:39+00:00" + "time": "2023-12-10T21:03:43+00:00" }, { "name": "phar-io/manifest", @@ -3891,16 +3891,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.24.4", + "version": "1.24.5", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "6bd0c26f3786cd9b7c359675cb789e35a8e07496" + "reference": "fedf211ff14ec8381c9bf5714e33a7a552dd1acc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/6bd0c26f3786cd9b7c359675cb789e35a8e07496", - "reference": "6bd0c26f3786cd9b7c359675cb789e35a8e07496", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/fedf211ff14ec8381c9bf5714e33a7a552dd1acc", + "reference": "fedf211ff14ec8381c9bf5714e33a7a552dd1acc", "shasum": "" }, "require": { @@ -3932,29 +3932,29 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.4" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.5" }, - "time": "2023-11-26T18:29:22+00:00" + "time": "2023-12-16T09:33:33+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.29", + "version": "9.2.30", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "6a3a87ac2bbe33b25042753df8195ba4aa534c76" + "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/6a3a87ac2bbe33b25042753df8195ba4aa534c76", - "reference": "6a3a87ac2bbe33b25042753df8195ba4aa534c76", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/ca2bd87d2f9215904682a9cb9bb37dda98e76089", + "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.15", + "nikic/php-parser": "^4.18 || ^5.0", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -4004,7 +4004,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.29" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.30" }, "funding": [ { @@ -4012,7 +4012,7 @@ "type": "github" } ], - "time": "2023-09-19T04:57:46+00:00" + "time": "2023-12-22T06:47:57+00:00" }, { "name": "phpunit/php-file-iterator", @@ -4651,20 +4651,20 @@ }, { "name": "sebastian/complexity", - "version": "2.0.2", + "version": "2.0.3", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/complexity.git", - "reference": "739b35e53379900cc9ac327b2147867b8b6efd88" + "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/739b35e53379900cc9ac327b2147867b8b6efd88", - "reference": "739b35e53379900cc9ac327b2147867b8b6efd88", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/25f207c40d62b8b7aa32f5ab026c53561964053a", + "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a", "shasum": "" }, "require": { - "nikic/php-parser": "^4.7", + "nikic/php-parser": "^4.18 || ^5.0", "php": ">=7.3" }, "require-dev": { @@ -4696,7 +4696,7 @@ "homepage": "https://github.com/sebastianbergmann/complexity", "support": { "issues": "https://github.com/sebastianbergmann/complexity/issues", - "source": "https://github.com/sebastianbergmann/complexity/tree/2.0.2" + "source": "https://github.com/sebastianbergmann/complexity/tree/2.0.3" }, "funding": [ { @@ -4704,7 +4704,7 @@ "type": "github" } ], - "time": "2020-10-26T15:52:27+00:00" + "time": "2023-12-22T06:19:30+00:00" }, { "name": "sebastian/diff", @@ -4978,20 +4978,20 @@ }, { "name": "sebastian/lines-of-code", - "version": "1.0.3", + "version": "1.0.4", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/lines-of-code.git", - "reference": "c1c2e997aa3146983ed888ad08b15470a2e22ecc" + "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/c1c2e997aa3146983ed888ad08b15470a2e22ecc", - "reference": "c1c2e997aa3146983ed888ad08b15470a2e22ecc", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/e1e4a170560925c26d424b6a03aed157e7dcc5c5", + "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5", "shasum": "" }, "require": { - "nikic/php-parser": "^4.6", + "nikic/php-parser": "^4.18 || ^5.0", "php": ">=7.3" }, "require-dev": { @@ -5023,7 +5023,7 @@ "homepage": "https://github.com/sebastianbergmann/lines-of-code", "support": { "issues": "https://github.com/sebastianbergmann/lines-of-code/issues", - "source": "https://github.com/sebastianbergmann/lines-of-code/tree/1.0.3" + "source": "https://github.com/sebastianbergmann/lines-of-code/tree/1.0.4" }, "funding": [ { @@ -5031,7 +5031,7 @@ "type": "github" } ], - "time": "2020-11-28T06:42:11+00:00" + "time": "2023-12-22T06:20:34+00:00" }, { "name": "sebastian/object-enumerator", From 1da476e5acf1e52342d2371914abb923ac73f389 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 9 Jan 2024 11:13:58 +0200 Subject: [PATCH 032/406] remove cloud related scripts --- Dockerfile | 14 +- bin/calc-tier-stats | 3 - bin/calc-users-stats | 3 - bin/clear-card-cache | 3 - bin/delete-orphaned-projects | 3 - bin/get-migration-stats | 3 - bin/hamster | 3 - bin/patch-delete-project-collections | 3 - ...patch-delete-schedule-updated-at-attribute | 3 - bin/patch-recreate-repositories-documents | 3 - bin/volume-sync | 3 - bin/worker-hamster | 3 - docker-compose.yml | 57 --- src/Appwrite/Event/Event.php | 3 - src/Appwrite/Event/Hamster.php | 157 ------- src/Appwrite/Platform/Services/Tasks.php | 11 - src/Appwrite/Platform/Tasks/CalcTierStats.php | 359 -------------- .../Platform/Tasks/DeleteOrphanedProjects.php | 161 ------- .../Platform/Tasks/GetMigrationStats.php | 187 -------- src/Appwrite/Platform/Tasks/Hamster.php | 158 ------- .../PatchRecreateRepositoriesDocuments.php | 169 ------- src/Appwrite/Platform/Tasks/VolumeSync.php | 59 --- src/Appwrite/Platform/Workers/Hamster.php | 437 ------------------ 23 files changed, 1 insertion(+), 1804 deletions(-) delete mode 100644 bin/calc-tier-stats delete mode 100644 bin/calc-users-stats delete mode 100644 bin/clear-card-cache delete mode 100644 bin/delete-orphaned-projects delete mode 100644 bin/get-migration-stats delete mode 100644 bin/hamster delete mode 100644 bin/patch-delete-project-collections delete mode 100644 bin/patch-delete-schedule-updated-at-attribute delete mode 100644 bin/patch-recreate-repositories-documents delete mode 100644 bin/volume-sync delete mode 100644 bin/worker-hamster delete mode 100644 src/Appwrite/Event/Hamster.php delete mode 100644 src/Appwrite/Platform/Tasks/CalcTierStats.php delete mode 100644 src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php delete mode 100644 src/Appwrite/Platform/Tasks/GetMigrationStats.php delete mode 100644 src/Appwrite/Platform/Tasks/Hamster.php delete mode 100644 src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php delete mode 100644 src/Appwrite/Platform/Tasks/VolumeSync.php delete mode 100644 src/Appwrite/Platform/Workers/Hamster.php diff --git a/Dockerfile b/Dockerfile index 2f85f2cc43..4fba47cdbf 100755 --- a/Dockerfile +++ b/Dockerfile @@ -94,20 +94,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ - chmod +x /usr/local/bin/worker-migrations && \ - chmod +x /usr/local/bin/worker-hamster + chmod +x /usr/local/bin/worker-migrations -# Cloud Executabless -RUN chmod +x /usr/local/bin/hamster && \ - chmod +x /usr/local/bin/volume-sync && \ - chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ - chmod +x /usr/local/bin/patch-recreate-repositories-documents && \ - chmod +x /usr/local/bin/patch-delete-project-collections && \ - chmod +x /usr/local/bin/delete-orphaned-projects && \ - chmod +x /usr/local/bin/clear-card-cache && \ - chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats && \ - chmod +x /usr/local/bin/get-migration-stats # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/calc-tier-stats b/bin/calc-tier-stats deleted file mode 100644 index c7fb71e6fd..0000000000 --- a/bin/calc-tier-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php calc-tier-stats $@ \ No newline at end of file diff --git a/bin/calc-users-stats b/bin/calc-users-stats deleted file mode 100644 index 60472ed7a2..0000000000 --- a/bin/calc-users-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php calc-users-stats $@ \ No newline at end of file diff --git a/bin/clear-card-cache b/bin/clear-card-cache deleted file mode 100644 index b39bc13651..0000000000 --- a/bin/clear-card-cache +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php clear-card-cache $@ \ No newline at end of file diff --git a/bin/delete-orphaned-projects b/bin/delete-orphaned-projects deleted file mode 100644 index 16b9e3184d..0000000000 --- a/bin/delete-orphaned-projects +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php delete-orphaned-projects $@ \ No newline at end of file diff --git a/bin/get-migration-stats b/bin/get-migration-stats deleted file mode 100644 index efc1934e15..0000000000 --- a/bin/get-migration-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php get-migration-stats $@ \ No newline at end of file diff --git a/bin/hamster b/bin/hamster deleted file mode 100644 index dcc7ed308d..0000000000 --- a/bin/hamster +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php hamster $@ \ No newline at end of file diff --git a/bin/patch-delete-project-collections b/bin/patch-delete-project-collections deleted file mode 100644 index 8bf0736cf3..0000000000 --- a/bin/patch-delete-project-collections +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-delete-project-collections $@ \ No newline at end of file diff --git a/bin/patch-delete-schedule-updated-at-attribute b/bin/patch-delete-schedule-updated-at-attribute deleted file mode 100644 index 3e28289cbe..0000000000 --- a/bin/patch-delete-schedule-updated-at-attribute +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@ \ No newline at end of file diff --git a/bin/patch-recreate-repositories-documents b/bin/patch-recreate-repositories-documents deleted file mode 100644 index 8c6c4157f4..0000000000 --- a/bin/patch-recreate-repositories-documents +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@ \ No newline at end of file diff --git a/bin/volume-sync b/bin/volume-sync deleted file mode 100644 index 5190750a24..0000000000 --- a/bin/volume-sync +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php volume-sync $@ \ No newline at end of file diff --git a/bin/worker-hamster b/bin/worker-hamster deleted file mode 100644 index b388dd13c9..0000000000 --- a/bin/worker-hamster +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php hamster $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 97cf7e5136..42091e5e46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -717,63 +717,6 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY - appwrite-worker-hamster: - entrypoint: worker-hamster - <<: *x-logging - container_name: appwrite-worker-hamster - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_MIXPANEL_TOKEN - - appwrite-hamster-scheduler: - entrypoint: hamster - <<: *x-logging - container_name: appwrite-hamster-scheduler - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_HAMSTER_TIME - - _APP_HAMSTER_INTERVAL - openruntimes-executor: container_name: openruntimes-executor hostname: appwrite-executor diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index fc12c5b5b3..46b430d122 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -42,9 +42,6 @@ class Event public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; - public const HAMSTER_QUEUE_NAME = 'v1-hamster'; - public const HAMSTER_CLASS_NAME = 'HamsterV1'; - protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php deleted file mode 100644 index 5d79fce568..0000000000 --- a/src/Appwrite/Event/Hamster.php +++ /dev/null @@ -1,157 +0,0 @@ -setQueue(Event::HAMSTER_QUEUE_NAME) - ->setClass(Event::HAMSTER_CLASS_NAME); - } - - /** - * Sets the type for the hamster event. - * - * @param string $type - * @return self - */ - public function setType(string $type): self - { - $this->type = $type; - - return $this; - } - - /** - * Returns the set type for the hamster event. - * - * @return string - */ - public function getType(): string - { - return $this->type; - } - - /** - * Sets the project for the hamster event. - * - * @param Document $project - */ - public function setProject(Document $project): self - { - $this->project = $project; - - return $this; - } - - /** - * Returns the set project for the hamster event. - * - * @return Document - */ - public function getProject(): Document - { - return $this->project; - } - - /** - * Sets the organization for the hamster event. - * - * @param Document $organization - */ - public function setOrganization(Document $organization): self - { - $this->organization = $organization; - - return $this; - } - - /** - * Returns the set organization for the hamster event. - * - * @return string - */ - public function getOrganization(): Document - { - return $this->organization; - } - - /** - * Sets the user for the hamster event. - * - * @param Document $user - */ - public function setUser(Document $user): self - { - $this->user = $user; - - return $this; - } - - /** - * Returns the set user for the hamster event. - * - * @return Document - */ - public function getUser(): Document - { - return $this->user; - } - - /** - * Executes the function event and sends it to the functions worker. - * - * @return string|bool - * @throws \InvalidArgumentException - */ - public function trigger(): string|bool - { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; - - return $client->enqueue([ - 'type' => $this->type, - 'project' => $this->project, - 'organization' => $this->organization, - 'user' => $this->user, - 'events' => $events, - ]); - } - - /** - * Generate a function event from a base event - * - * @param Event $event - * - * @return self - * - */ - public function from(Event $event): self - { - $this->event = $event->getEvent(); - $this->params = $event->getParams(); - return $this; - } -} diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index dc6ddc1a5b..0da42e3545 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -15,12 +15,7 @@ use Appwrite\Platform\Tasks\Hamster; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; -use Appwrite\Platform\Tasks\VolumeSync; -use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; -use Appwrite\Platform\Tasks\DeleteOrphanedProjects; -use Appwrite\Platform\Tasks\GetMigrationStats; -use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service { @@ -40,13 +35,7 @@ class Tasks extends Service ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) - ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) - ->addAction(CalcTierStats::getName(), new CalcTierStats()) - ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) - ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) - ->addAction(GetMigrationStats::getName(), new GetMigrationStats()) - ; } } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php deleted file mode 100644 index e6559a05d7..0000000000 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ /dev/null @@ -1,359 +0,0 @@ - 'Requests', - 'project.$all.network.bandwidth' => 'Bandwidth', - - ]; - - public static function getName(): string - { - return 'calc-tier-stats'; - } - - public function __construct() - { - - $this - ->desc('Get stats for projects') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); - }); - } - - /** - * @throws \Utopia\Exception - * @throws CannotInsertRecord - */ - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - //docker compose exec -t appwrite calc-tier-stats - - Console::title('Cloud free tier stats calculation V1'); - Console::success(APP_NAME . ' cloud free tier stats calculation has started'); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/tier_stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $projects = [$console]; - $count = 0; - $limit = 100; - $sum = 100; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - /** Get Project ID */ - $stats['Project ID'] = $project->getId(); - - $stats['Organization ID'] = $project->getAttribute('teamId', null); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $stats['Organization Members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ]); - } else { - $stats['Organization Members'] = 0; - } - - /** Get Total internal Teams */ - try { - $stats['Teams'] = $dbForProject->count('teams', []); - } catch (\Throwable) { - $stats['Teams'] = 0; - } - - /** Get Total users */ - try { - $stats['Users'] = $dbForProject->count('users', []); - } catch (\Throwable) { - $stats['Users'] = 0; - } - - /** Get Usage stats */ - $range = '30d'; - $periods = [ - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ] - ]; - - $tmp = []; - $metrics = $this->usageStats; - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) { - foreach ($metrics as $metric => $name) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $tmp[$metric] = []; - foreach ($requestDocs as $requestDoc) { - if (empty($requestDoc)) { - continue; - } - - $tmp[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $tmp[$metric] = array_reverse($tmp[$metric]); - $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value')); - } - }); - - foreach ($tmp as $key => $value) { - $stats[$metrics[$key]] = $value; - } - - try { - /** Get Domains */ - $stats['Domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Domains'] = 0; - } - - try { - /** Get Api keys */ - $stats['Api keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Api keys'] = 0; - } - - try { - /** Get Webhooks */ - $stats['Webhooks'] = $dbForConsole->count('webhooks', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Webhooks'] = 0; - } - - try { - /** Get Platforms */ - $stats['Platforms'] = $dbForConsole->count('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Platforms'] = 0; - } - - /** Get Files & Buckets */ - $filesCount = 0; - $filesSum = 0; - $maxFileSize = 0; - $counter = 0; - try { - $buckets = $dbForProject->find('buckets', []); - foreach ($buckets as $bucket) { - $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]); - if (empty($file)) { - continue; - } - $filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []); - $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []); - if ($file->getAttribute('sizeOriginal') > $maxFileSize) { - $maxFileSize = $file->getAttribute('sizeOriginal'); - } - $counter++; - } - } catch (\Throwable) { - ; - } - $stats['Buckets'] = $counter; - $stats['Files'] = $filesCount; - $stats['Storage (bytes)'] = $filesSum; - $stats['Max File Size (bytes)'] = $maxFileSize; - - - try { - /** Get Total Functions */ - $stats['Databases'] = $dbForProject->count('databases', []); - } catch (\Throwable) { - $stats['Databases'] = 0; - } - - /** Get Total Functions */ - try { - $stats['Functions'] = $dbForProject->count('functions', []); - } catch (\Throwable) { - $stats['Functions'] = 0; - } - - /** Get Total Deployments */ - try { - $stats['Deployments'] = $dbForProject->count('deployments', []); - } catch (\Throwable) { - $stats['Deployments'] = 0; - } - - /** Get Total Executions */ - try { - $stats['Executions'] = $dbForProject->count('executions', []); - } catch (\Throwable) { - $stats['Executions'] = 0; - } - - /** Get Total Migrations */ - try { - $stats['Migrations'] = $dbForProject->count('migrations', []); - } catch (\Throwable) { - $stats['Migrations'] = 0; - } - - $csv->insertOne(array_values($stats)); - } catch (\Throwable $th) { - Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); - - $pools - ->get('console') - ->reclaim(); - - /** @var PHPMailer $mail */ - $mail = $register->get('smtp'); - - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - - try { - /** Addresses */ - $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); - $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); - - foreach ($recipients as $recipient) { - $mail->addAddress($recipient); - } - - /** Attachments */ - $mail->addAttachment($this->path); - - /** Content */ - $mail->Subject = "Cloud Report for {$this->date}"; - $mail->Body = "Please find the daily cloud report atttached"; - $mail->send(); - Console::success('Email has been sent!'); - } catch (Exception $e) { - Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - } - } -} diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php deleted file mode 100644 index 757b29c1b6..0000000000 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ /dev/null @@ -1,161 +0,0 @@ -desc('Delete orphaned projects') - ->param('commit', false, new Boolean(true), 'Commit project deletion', true) - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($commit, $pools, $cache, $dbForConsole, $register); - }); - } - - - public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - - Console::title('Delete orphaned projects V1'); - Console::success(APP_NAME . ' Delete orphaned projects started'); - - /** @var array $collections */ - $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; - - $collectionsConfig = array_merge([ - 'audit' => [ - '$id' => ID::custom('audit'), - '$collection' => Database::METADATA - ], - 'abuse' => [ - '$id' => ID::custom('abuse'), - '$collection' => Database::METADATA - ] - ], $collectionsConfig); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - $projects = [$console]; - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $orphans = 1; - $cnt = 0; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $collectionsCreated = 0; - $cnt++; - if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { - $collectionsCreated = $dbForProject->count(Database::METADATA); - } - - $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; - - if ($collectionsCreated >= count($collectionsConfig)) { - Console::log($msg . ' ignoring....'); - continue; - } - - Console::log($msg); - - if ($collectionsCreated > 0) { - $collections = $dbForProject->find(Database::METADATA, []); - foreach ($collections as $collection) { - if ($commit) { - $dbForProject->deleteCollection($collection->getId()); - $dbForConsole->deleteCachedCollection($collection->getId()); - } - Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); - } - } - - if ($commit) { - $dbForConsole->deleteDocument('projects', $project->getId()); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); - - if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { - try { - $dbForProject->deleteCollection(Database::METADATA); - $dbForProject->deleteCachedCollection(Database::METADATA); - } catch (\Throwable $th) { - Console::warning('Metadata collection does not exist'); - } - } - } - - Console::info('--Deleting project no (' . $project->getInternalId() . ')'); - - $orphans++; - } catch (\Throwable $th) { - Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); - } -} diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php deleted file mode 100644 index b76e0428d7..0000000000 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ /dev/null @@ -1,187 +0,0 @@ -desc('Get stats for projects') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); - }); - } - - /** - * @throws \Utopia\Exception - * @throws CannotInsertRecord - */ - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - //docker compose exec -t appwrite get-migration-stats - - Console::title('Migration stats calculation V1'); - Console::success(APP_NAME . ' Migration stats calculation has started'); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/migration_stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $projects = [$console]; - $count = 0; - $limit = 100; - $sum = 100; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - /** Get Project ID */ - $stats['Project ID'] = $project->getId(); - - /** Get Migration details */ - $migrations = $dbForProject->find('migrations', [ - Query::limit(500) - ]); - - $migrations = array_map(function ($migration) use ($project) { - return [ - $project->getId(), - $migration->getAttribute('$id'), - $migration->getAttribute('$createdAt'), - $migration->getAttribute('status'), - $migration->getAttribute('stage'), - $migration->getAttribute('source'), - ]; - }, $migrations); - - if (!empty($migrations)) { - $csv->insertAll($migrations); - } - } catch (\Throwable $th) { - Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); - - $pools - ->get('console') - ->reclaim(); - - /** @var PHPMailer $mail */ - $mail = $register->get('smtp'); - - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - - try { - /** Addresses */ - $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); - $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); - - foreach ($recipients as $recipient) { - $mail->addAddress($recipient); - } - - /** Attachments */ - $mail->addAttachment($this->path); - - /** Content */ - $mail->Subject = "Migration Report for {$this->date}"; - $mail->Body = "Please find the migration report atttached"; - $mail->send(); - Console::success('Email has been sent!'); - } catch (Exception $e) { - Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - } - } -} diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php deleted file mode 100644 index 1dca095e93..0000000000 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ /dev/null @@ -1,158 +0,0 @@ -desc('Get stats for projects') - ->inject('queueForHamster') - ->inject('dbForConsole') - ->callback(function (EventHamster $queueForHamster, Database $dbForConsole) { - $this->action($queueForHamster, $dbForConsole); - }); - } - - public function action(EventHamster $queueForHamster, Database $dbForConsole): void - { - Console::title('Cloud Hamster V1'); - Console::success(APP_NAME . ' cloud hamster process has started'); - - $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) - - $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes) - - $now = new \DateTime(); - $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); - - $next = new \DateTime($now->format("Y-m-d $jobInitTime")); - $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); - - $delay = $next->getTimestamp() - $now->getTimestamp(); - /** - * If time passed for the target day. - */ - if ($delay <= 0) { - $next->add(\DateInterval::createFromDateString('1 days')); - $delay = $next->getTimestamp() - $now->getTimestamp(); - } - - Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']'); - - Console::loop(function () use ($queueForHamster, $dbForConsole, $sleep) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Queuing Cloud Usage Stats every {$sleep} seconds"); - $loopStart = microtime(true); - - Console::info('Queuing stats for all projects'); - $this->getStatsPerProject($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all projects'); - - Console::info('Queuing stats for all organizations'); - $this->getStatsPerOrganization($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all organizations'); - - Console::info('Queuing stats for all users'); - $this->getStatsPerUser($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all users'); - - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); - }, $sleep, $delay); - } - - protected function calculateByGroup(string $collection, Database $database, callable $callback) - { - $count = 0; - $chunk = 0; - $limit = 50; - $results = []; - $sum = $limit; - - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([ - Query::limit($limit), - Query::offset($count) - ])); - - $sum = count($results); - - Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents'); - - foreach ($results as $document) { - call_user_func($callback, $database, $document); - $count++; - } - } - - $executionEnd = \microtime(true); - - Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); - } - - protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster, $loopStart) { - try { - $organization->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_ORGANISATION) - ->setOrganization($organization) - ->trigger(); - } catch (Exception $e) { - Console::error($e->getMessage()); - } - }); - } - - private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster, $loopStart) { - try { - $project->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_PROJECT) - ->setProject($project) - ->trigger(); - } catch (Exception $e) { - Console::error($e->getMessage()); - } - }); - } - - protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster, $loopStart) { - try { - $user->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_USER) - ->setUser($user) - ->trigger(); - } catch (Exception $e) { - Console::error($e->getMessage()); - } - }); - } -} diff --git a/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php deleted file mode 100644 index 93e6c527bb..0000000000 --- a/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php +++ /dev/null @@ -1,169 +0,0 @@ -desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.') - ->param('after', '', new Text(36), 'After cursor', true) - ->param('projectId', '', new Text(36), 'Select project to validate', true) - ->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB)); - } - - public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void - { - Console::info("Starting the patch"); - - $startTime = microtime(true); - - if (!empty($projectId)) { - try { - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occured with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - } else { - $queries = []; - if (!empty($after)) { - Console::info("Iterating remaining projects after project with ID {$after}"); - $project = $dbForConsole->getDocument('projects', $after); - $queries = [Query::cursorAfter($project)]; - } else { - Console::info("Iterating all projects"); - } - $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) { - $projectId = $project->getId(); - - try { - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occured with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - }); - } - - $endTime = microtime(true); - $timeTaken = $endTime - $startTime; - - $hours = (int)($timeTaken / 3600); - $timeTaken -= $hours * 3600; - $minutes = (int)($timeTaken / 60); - $timeTaken -= $minutes * 60; - $seconds = (int)$timeTaken; - $milliseconds = ($timeTaken - $seconds) * 1000; - Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)"); - } - - protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void - { - $limit = 1000; - $results = []; - $sum = $limit; - $latestDocument = null; - - while ($sum === $limit) { - $newQueries = $queries; - - if ($latestDocument != null) { - array_unshift($newQueries, Query::cursorAfter($latestDocument)); - } - $newQueries[] = Query::limit($limit); - $results = $database->find($collection, $newQueries); - - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void - { - $projectId = $project->getId(); - Console::log("Running patch for project {$projectId}"); - - $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) { - $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); - - if ($isConnected) { - $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); - - if ($repository->isEmpty()) { - $projectId = $project->getId(); - $functionId = $function->getId(); - Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); - - $repository = $dbForConsole->createDocument('repositories', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'installationId' => $function->getAttribute('installationId', ''), - 'installationInternalId' => $function->getAttribute('installationInternalId', ''), - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getInternalId(), - 'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''), - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getInternalId(), - 'resourceType' => 'function', - 'providerPullRequestIds' => [] - ])); - - $function = $dbForProject->updateDocument('functions', $function->getId(), $function - ->setAttribute('repositoryId', $repository->getId()) - ->setAttribute('repositoryInternalId', $repository->getInternalId())); - - $this->foreachDocument($dbForProject, 'deployments', [ - Query::equal('resourceInternalId', [$function->getInternalId()]), - Query::equal('resourceType', ['functions']) - ], function (Document $deployment) use ($dbForProject, $repository) { - $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment - ->setAttribute('repositoryId', $repository->getId()) - ->setAttribute('repositoryInternalId', $repository->getInternalId())); - }); - } - } - }); - } -} diff --git a/src/Appwrite/Platform/Tasks/VolumeSync.php b/src/Appwrite/Platform/Tasks/VolumeSync.php deleted file mode 100644 index 6197b20fbd..0000000000 --- a/src/Appwrite/Platform/Tasks/VolumeSync.php +++ /dev/null @@ -1,59 +0,0 @@ -desc('Runs rsync to sync certificates between the storage mount and traefik.') - ->param('source', null, new Text(255), 'Source path to sync from.', false) - ->param('destination', null, new Text(255), 'Destination path to sync to.', false) - ->param('interval', null, new Integer(true), 'Interval to run rsync', false) - ->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval)); - } - - public function action(string $source, string $destination, int $interval) - { - - Console::title('RSync V1'); - Console::success(APP_NAME . ' rsync process v1 has started'); - - if (!file_exists($source)) { - Console::error('Source directory does not exist. Exiting ... '); - Console::exit(0); - } - - Console::loop(function () use ($interval, $source, $destination) { - $time = DateTime::now(); - - Console::info("[{$time}] Executing rsync every {$interval} seconds"); - Console::info("Syncing between $source and $destination"); - - if (!file_exists($source)) { - Console::error('Source directory does not exist. Skipping ... '); - return; - } - - $stdin = ""; - $stdout = ""; - $stderr = ""; - - Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr); - Console::success($stdout); - Console::error($stderr); - }, $interval); - } -} diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php deleted file mode 100644 index e911bb6c7a..0000000000 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ /dev/null @@ -1,437 +0,0 @@ - 'files.$all.count.total', - 'usage_buckets' => 'buckets.$all.count.total', - 'usage_databases' => 'databases.$all.count.total', - 'usage_documents' => 'documents.$all.count.total', - 'usage_collections' => 'collections.$all.count.total', - 'usage_storage' => 'project.$all.storage.size', - 'usage_requests' => 'project.$all.network.requests', - 'usage_bandwidth' => 'project.$all.network.bandwidth', - 'usage_users' => 'users.$all.count.total', - 'usage_sessions' => 'sessions.email.requests.create', - 'usage_executions' => 'executions.$all.compute.total', - ]; - - protected Mixpanel $mixpanel; - - public static function getName(): string - { - return 'hamster'; - } - - /** - * @throws \Exception - */ - public function __construct() - { - $this - ->desc('Hamster worker') - ->inject('message') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole) => $this->action($message, $pools, $cache, $dbForConsole)); - } - - /** - * @param Message $message - * @param Group $pools - * @param Cache $cache - * @param Database $dbForConsole - * - * @return void - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole): void - { - $token = App::getEnv('_APP_MIXPANEL_TOKEN', ''); - if (empty($token)) { - throw new \Exception('Missing MixPanel Token'); - } - $this->mixpanel = new Mixpanel($token); - - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new \Exception('Missing payload'); - } - - $type = $payload['type'] ?? ''; - - switch ($type) { - case EventHamster::TYPE_PROJECT: - $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole); - break; - case EventHamster::TYPE_ORGANISATION: - $this->getStatsForOrganization(new Document($payload['organization']), $dbForConsole); - break; - case EventHamster::TYPE_USER: - $this->getStatsPerUser(new Document($payload['user']), $dbForConsole); - break; - } - } - - /** - * @param Document $project - * @param Group $pools - * @param Cache $cache - * @param Database $dbForConsole - * @throws \Utopia\Database\Exception - */ - private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole): void - { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - Console::info("Skipping project console"); - return; - } - - Console::log("Getting stats for Project {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $statsPerProject = []; - - $statsPerProject['time'] = $project->getAttribute('$time'); - - /** Get Project ID */ - $statsPerProject['project_id'] = $project->getId(); - - /** Get project created time */ - $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); - - /** Get Project Name */ - $statsPerProject['project_name'] = $project->getAttribute('name'); - - /** Total Project Variables */ - $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); - - /** Total Migrations */ - $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); - - /** Get Custom SMTP */ - $smtp = $project->getAttribute('smtp', null); - if ($smtp) { - $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; - - /** Get Custom Templates Count */ - $templates = array_keys($project->getAttribute('templates', [])); - $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'email'); - }); - $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'sms'); - }); - } - - /** Get total relationship attributes */ - $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ - Query::equal('type', ['relationship']) - ], APP_LIMIT_COUNT); - - /** Get Total Functions */ - $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - - foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { - $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ - Query::equal('runtime', [$runtime]), - ], APP_LIMIT_COUNT); - } - - /** Get Total Deployments */ - $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ - Query::equal('type', ['manual']) - ], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ - Query::equal('type', ['vcs']) - ], APP_LIMIT_COUNT); - - /** Get VCS repos connected */ - $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ], APP_LIMIT_COUNT); - - /** Get Total Teams */ - $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ], APP_LIMIT_COUNT); - } else { - $statsPerProject['custom_organization_members'] = 0; - } - - /** Get Email and Name of the project owner */ - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new \Exception('Membership not found. Skipping project : ' . $project->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerProject['email'] = $user->getAttribute('email', null); - $statsPerProject['name'] = $user->getAttribute('name', null); - } - } - - /** Get Domains */ - $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Platforms */ - $platforms = $dbForConsole->find('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'web'; - })); - - $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'android'; - })); - - $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'apple'); - })); - - $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'flutter'); - })); - - $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; - - foreach ($flutterPlatforms as $flutterPlatform) { - $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { - return $platform['type'] === $flutterPlatform; - })); - } - - $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Usage $statsPerProject */ - $periods = [ - 'infinity' => [ - 'period' => '1d', - 'limit' => 90, - ], - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - ]; - - Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { - foreach ($this->metrics as $key => $metric) { - foreach ($periods as $periodKey => $periodValue) { - $limit = $periodValue['limit']; - $period = $periodValue['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $statsPerProject[$key . '_' . $periodKey] = []; - foreach ($requestDocs as $requestDoc) { - $statsPerProject[$key . '_' . $periodKey][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); - // Calculate aggregate of each metric - $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); - } - } - }); - - if (isset($statsPerProject['email'])) { - /** Send data to mixpanel */ - $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ - 'name' => $statsPerProject['name'], - 'email' => $statsPerProject['email'] - ]); - - if (!$res) { - Console::error('Failed to create user profile for project: ' . $project->getId()); - } - } - - $event = new AnalyticsEvent(); - $event - ->setName('Project Daily Usage') - ->setProps($statsPerProject); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - Console::error('Failed to create event for project: ' . $project->getId()); - } - } catch (\Exception $e) { - Console::error('Failed to send stats for project: ' . $project->getId()); - Console::error($e->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - /** - * @param Document $organization - * @param Database $dbForConsole - * @throws \Utopia\Database\Exception - */ - private function getStatsForOrganization(Document $organization, Database $dbForConsole): void - { - Console::log("Getting stats for Organization {$organization->getId()}"); - - try { - $statsPerOrganization = []; - - $statsPerOrganization['time'] = $organization->getAttribute('$time'); - - /** Organization name */ - $statsPerOrganization['name'] = $organization->getAttribute('name'); - - - /** Get Email and of the organization owner */ - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$organization->getInternalId()]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerOrganization['email'] = $user->getAttribute('email', null); - } - - /** Organization Creation Date */ - $statsPerOrganization['created'] = $organization->getAttribute('$createdAt'); - - /** Number of team members */ - $statsPerOrganization['members'] = $organization->getAttribute('total'); - - /** Number of projects in this organization */ - $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ - Query::equal('teamId', [$organization->getId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerOrganization['email'])) { - throw new \Exception('Email not found. Skipping organization : ' . $organization->getId()); - } - - $event = new AnalyticsEvent(); - $event - ->setName('Organization Daily Usage') - ->setProps($statsPerOrganization); - $res = $this->mixpanel->createEvent($event); - if (!$res) { - throw new \Exception('Failed to create event for organization : ' . $organization->getId()); - } - } catch (\Exception $e) { - Console::error($e->getMessage()); - } - } - - protected function getStatsPerUser(Document $user, Database $dbForConsole) - { - Console::log("Getting stats for User {$user->getId()}"); - - try { - $statsPerUser = []; - - $statsPerUser['time'] = $user->getAttribute('$time'); - - /** Organization name */ - $statsPerUser['name'] = $user->getAttribute('name'); - - /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ - $statsPerUser['email'] = $user->getAttribute('email'); - - /** Organization Creation Date */ - $statsPerUser['created'] = $user->getAttribute('$createdAt'); - - /** Number of teams this user is a part of */ - $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ - Query::equal('userInternalId', [$user->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerUser['email'])) { - throw new \Exception('User has no email: ' . $user->getId()); - } - - /** Send data to mixpanel */ - $event = new AnalyticsEvent(); - $event - ->setName('User Daily Usage') - ->setProps($statsPerUser); - - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - throw new \Exception('Failed to create user profile for user: ' . $user->getId()); - } - } catch (\Exception $e) { - Console::error($e->getMessage()); - } - } -} From eb70990bf01b77bf6586d131710b80620bfe3d47 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 9 Jan 2024 15:22:02 +0200 Subject: [PATCH 033/406] remove cloud related scripts --- src/Appwrite/Platform/Services/Tasks.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 0da42e3545..6b8a111611 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -11,7 +11,6 @@ use Appwrite\Platform\Tasks\Schedule; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; -use Appwrite\Platform\Tasks\Hamster; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; From 85950868f25308436eab7da3eb891b3ebed661c1 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 9 Jan 2024 15:30:43 +0200 Subject: [PATCH 034/406] remove cloud related scripts --- src/Appwrite/Platform/Services/Tasks.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 6b8a111611..808fb07a81 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -26,7 +26,6 @@ class Tasks extends Service ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) - ->addAction(Hamster::getName(), new Hamster()) ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Upgrade::getName(), new Upgrade()) From 4b23440c2ddb2c9a8527223f7611d68837df934e Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 9 Jan 2024 15:44:56 +0200 Subject: [PATCH 035/406] remove cloud related scripts --- src/Appwrite/Platform/Services/Workers.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index c5a0514760..40366f2313 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -12,7 +12,6 @@ use Appwrite\Platform\Workers\Databases; use Appwrite\Platform\Workers\Functions; use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; -use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -31,8 +30,6 @@ class Workers extends Service ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) ->addAction(Migrations::getName(), new Migrations()) - ->addAction(Hamster::getName(), new Hamster()) - ; } } From 6ad63b6f7bc58cf504c1d004db8b8afbfd8ce564 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 17 Jan 2024 10:44:16 +0200 Subject: [PATCH 036/406] addressing comments --- app/cli.php | 3 --- app/worker.php | 3 --- 2 files changed, 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index 003f3a1f79..6e16ab1ddb 100644 --- a/app/cli.php +++ b/app/cli.php @@ -155,9 +155,6 @@ CLI::setResource('queue', function (Group $pools) { CLI::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); -CLI::setResource('queueForHamster', function (Connection $queue) { - return new Hamster($queue); -}, ['queue']); CLI::setResource('queueForDeletes', function (Connection $queue) { return new Delete($queue); }, ['queue']); diff --git a/app/worker.php b/app/worker.php index 4f7355311e..a7396b1169 100644 --- a/app/worker.php +++ b/app/worker.php @@ -156,9 +156,6 @@ Server::setResource('queueForCertificates', function (Connection $queue) { Server::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); -Server::setResource('queueForHamster', function (Connection $queue) { - return new Hamster($queue); -}, ['queue']); Server::setResource('logger', function (Registry $register) { return $register->get('logger'); }, ['register']); From ab80999d1e9def7e3bc1483990b50e411a97ac34 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 31 Jan 2024 12:13:02 +0200 Subject: [PATCH 037/406] cleanup --- bin/create-inf-metric | 3 - src/Appwrite/Platform/Tasks/CalcTierStats.php | 0 .../Platform/Tasks/CreateInfMetric.php | 413 ------------------ 3 files changed, 416 deletions(-) delete mode 100644 bin/create-inf-metric delete mode 100644 src/Appwrite/Platform/Tasks/CalcTierStats.php delete mode 100644 src/Appwrite/Platform/Tasks/CreateInfMetric.php diff --git a/bin/create-inf-metric b/bin/create-inf-metric deleted file mode 100644 index ea7b2b4da0..0000000000 --- a/bin/create-inf-metric +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php create-inf-metric $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/Appwrite/Platform/Tasks/CreateInfMetric.php b/src/Appwrite/Platform/Tasks/CreateInfMetric.php deleted file mode 100644 index 49b852ff6f..0000000000 --- a/src/Appwrite/Platform/Tasks/CreateInfMetric.php +++ /dev/null @@ -1,413 +0,0 @@ -desc('Create infinity stats metric') - ->param('after', '', new Text(36), 'After cursor', true) - ->param('projectId', '', new Text(36), 'Select project to validate', true) - ->inject('getProjectDB') - ->inject('dbForConsole') - ->callback(function (string $after, string $projectId, callable $getProjectDB, Database $dbForConsole) { - $this->action($after, $projectId, $getProjectDB, $dbForConsole); - }); - } - - - /** - * @throws Exception - * @throws Exception\Timeout - * @throws Exception\Query - */ - public function action(string $after, string $projectId, callable $getProjectDB, Database $dbForConsole): void - { - - Console::title('Create infinity metric V1'); - Console::success(APP_NAME . ' Create infinity metric started'); - - if (!empty($projectId)) { - try { - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $this->getUsageData($dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occured with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - } else { - $queries = []; - if (!empty($after)) { - Console::info("Iterating remaining projects after project with ID {$after}"); - $project = $dbForConsole->getDocument('projects', $after); - $queries = [Query::cursorAfter($project)]; - } else { - Console::info("Iterating all projects"); - } - $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB) { - $projectId = $project->getId(); - - try { - $dbForProject = call_user_func($getProjectDB, $project); - $this->getUsageData($dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occured with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - }); - } - } - - /** - * @param Database $database - * @param string $collection - * @param array $queries - * @param callable|null $callback - * @return void - * @throws Exception - * @throws Exception\Query - * @throws Exception\Timeout - */ - private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void - { - $limit = 1000; - $results = []; - $sum = $limit; - $latestDocument = null; - - while ($sum === $limit) { - $newQueries = $queries; - - if ($latestDocument != null) { - array_unshift($newQueries, Query::cursorAfter($latestDocument)); - } - $newQueries[] = Query::limit($limit); - $results = $database->find($collection, $newQueries); - - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - - /** - * @param Database $dbForProject - * @param Document $project - * @return void - */ - private function getUsageData(Database $dbForProject, Document $project): void - { - try { - $this->network($dbForProject); - $this->sessions($dbForProject); - $this->users($dbForProject); - $this->teams($dbForProject); - $this->databases($dbForProject); - $this->functions($dbForProject); - $this->storage($dbForProject); - } catch (\Throwable $th) { - var_dump($th->getMessage()); - } - - Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId()); - } - - - /** - * @param Database $dbForProject - * @param string $metric - * @param int|float $value - * @return void - * @throws Exception - * @throws Exception\Authorization - * @throws Exception\Conflict - * @throws Exception\Restricted - * @throws Exception\Structure - */ - private function createInfMetric(database $dbForProject, string $metric, int|float $value): void - { - - try { - $id = \md5("_inf_{$metric}"); - $dbForProject->deleteDocument('stats_v2', $id); - $dbForProject->createDocument('stats_v2', new Document([ - '$id' => $id, - 'metric' => $metric, - 'period' => 'inf', - 'value' => (int)$value, - 'time' => null, - 'region' => 'default', - ])); - } catch (Duplicate $th) { - console::log("Error while creating inf metric: duplicate id {$metric} {$id}"); - } - } - - /** - * @param Database $dbForProject - * @param string $metric - * @return int|float - * @throws Exception - */ - protected function getFromMetric(database $dbForProject, string $metric): int|float - { - - return $dbForProject->sum('stats_v2', 'value', [ - Query::equal('metric', [ - $metric, - ]), - Query::equal('period', ['1d']), - ]); - } - - /** - * @param Database $dbForProject - * @throws Exception - * @throws Exception\Authorization - * @throws Exception\Conflict - * @throws Exception\Restricted - * @throws Exception\Structure - */ - private function network(database $dbForProject) - { - $this->createInfMetric($dbForProject, 'network.inbound', $this->getFromMetric($dbForProject, 'network.inbound')); - $this->createInfMetric($dbForProject, 'network.outbound', $this->getFromMetric($dbForProject, 'network.outbound')); - $this->createInfMetric($dbForProject, 'network.requests', $this->getFromMetric($dbForProject, 'network.requests')); - } - - - /** - * @throws Exception\Authorization - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception\Timeout - * @throws Exception\Structure - * @throws Exception - * @throws Exception\Query - */ - private function storage(database $dbForProject) - { - $bucketsCount = 0; - $filesCount = 0; - $filesStorageSum = 0; - - $buckets = $dbForProject->find('buckets'); - foreach ($buckets as $bucket) { - $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); - $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $files); - - $filesStorage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal'); - $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStorage); - - $bucketsCount++; - $filesCount += $files; - $filesStorageSum += $filesStorage; - } - - $this->createInfMetric($dbForProject, 'buckets', $bucketsCount); - $this->createInfMetric($dbForProject, 'files', $filesCount); - $this->createInfMetric($dbForProject, 'files.storage', $filesStorageSum); - } - - - /** - * @throws Exception\Authorization - * @throws Exception\Timeout - * @throws Exception\Restricted - * @throws Exception\Structure - * @throws Exception\Conflict - * @throws Exception - * @throws Exception\Query - */ - private function functions(Database $dbForProject) - { - $functionsCount = 0; - $deploymentsCount = 0; - $buildsCount = 0; - $buildsStorageSum = 0; - $buildsComputeSum = 0; - $executionsCount = 0; - $executionsComputeSum = 0; - $deploymentsStorageSum = 0; - - //functions - $functions = $dbForProject->find('functions'); - foreach ($functions as $function) { - //deployments - $deployments = $dbForProject->find('deployments', [ - Query::equal('resourceType', ['functions']), - Query::equal('resourceInternalId', [$function->getInternalId()]), - ]); - - $deploymentCount = 0; - $deploymentStorageSum = 0; - foreach ($deployments as $deployment) { - //builds - $builds = $dbForProject->count('builds', [ - Query::equal('deploymentInternalId', [$deployment->getInternalId()]), - ]); - - $buildsCompute = $dbForProject->sum('builds', 'duration', [ - Query::equal('deploymentInternalId', [$deployment->getInternalId()]), - ]); - - $buildsStorage = $dbForProject->sum('builds', 'size', [ - Query::equal('deploymentInternalId', [$deployment->getInternalId()]), - ]); - - $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds', $builds); - $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.storage', $buildsCompute * 1000); - $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildsStorage); - - $buildsCount += $builds; - $buildsComputeSum += $buildsCompute; - $buildsStorageSum += $buildsStorage; - - - $deploymentCount++; - $deploymentsCount++; - $deploymentsStorageSum += $deployment['size']; - $deploymentStorageSum += $deployment['size']; - } - $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments', $deploymentCount); - $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments.storage', $deploymentStorageSum); - - //executions - $executions = $dbForProject->count('executions', [ - Query::equal('functionInternalId', [$function->getInternalId()]), - ]); - - $executionsCompute = $dbForProject->sum('executions', 'duration', [ - Query::equal('functionInternalId', [$function->getInternalId()]), - ]); - - $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executions); - $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', $executionsCompute * 1000); - $executionsCount += $executions; - $executionsComputeSum += $executionsCompute; - - $functionsCount++; - } - - $this->createInfMetric($dbForProject, 'functions', $functionsCount); - $this->createInfMetric($dbForProject, 'deployments', $deploymentsCount); - $this->createInfMetric($dbForProject, 'deployments.storage', $deploymentsStorageSum); - $this->createInfMetric($dbForProject, 'builds', $buildsCount); - $this->createInfMetric($dbForProject, 'builds.compute', $buildsComputeSum * 1000); - $this->createInfMetric($dbForProject, 'builds.storage', $buildsStorageSum); - $this->createInfMetric($dbForProject, 'executions', $executionsCount); - $this->createInfMetric($dbForProject, 'executions.compute', $executionsComputeSum * 1000); - } - - /** - * @throws Exception\Authorization - * @throws Exception\Timeout - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - * @throws Exception\Query - */ - private function databases(Database $dbForProject) - { - $databasesCount = 0; - $collectionsCount = 0; - $documentsCount = 0; - $databases = $dbForProject->find('databases'); - foreach ($databases as $database) { - $collectionCount = 0; - $collections = $dbForProject->find('database_' . $database->getInternalId()); - foreach ($collections as $collection) { - $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - $this->createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documents); - $documentsCount += $documents; - $collectionCount++; - $collectionsCount++; - } - $this->createInfMetric($dbForProject, $database->getInternalId() . '.collections', $collectionCount); - $this->createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCount); - $databasesCount++; - } - $this->createInfMetric($dbForProject, 'collections', $collectionsCount); - $this->createInfMetric($dbForProject, 'databases', $databasesCount); - $this->createInfMetric($dbForProject, 'documents', $documentsCount); - } - - - /** - * @throws Exception\Authorization - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - */ - private function users(Database $dbForProject) - { - $users = $dbForProject->count('users'); - $this->createInfMetric($dbForProject, 'users', $users); - } - - /** - * @throws Exception\Authorization - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - */ - private function sessions(Database $dbForProject) - { - $users = $dbForProject->count('sessions'); - $this->createInfMetric($dbForProject, 'sessions', $users); - } - - /** - * @throws Exception\Authorization - * @throws Exception\Structure - * @throws Exception\Restricted - * @throws Exception\Conflict - * @throws Exception - */ - private function teams(Database $dbForProject) - { - $teams = $dbForProject->count('teams'); - $this->createInfMetric($dbForProject, 'teams', $teams); - } -} From d4afe2239662927b06844bdffdcb2118facb48b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 6 Feb 2024 12:07:47 +0100 Subject: [PATCH 038/406] Add ZDT migration support to project creation --- app/controllers/api/projects.php | 10 +++++++++- src/Appwrite/Hooks/Hooks.php | 5 +++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 8314a21f30..135ce9a50f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -4,6 +4,7 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Delete; use Appwrite\Event\Validator\Event; use Appwrite\Extend\Exception; +use Appwrite\Hooks\Hooks; use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\Origin; use Appwrite\Template\Template; @@ -75,7 +76,8 @@ App::post('/v1/projects') ->inject('dbForConsole') ->inject('cache') ->inject('pools') - ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { + ->inject('hooks') + ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools, Hooks $hooks) { $team = $dbForConsole->getDocument('teams', $teamId); @@ -183,6 +185,12 @@ App::post('/v1/projects') /** @var array $collections */ $collections = Config::getParam('collections', [])['projects'] ?? []; + // Allow Cloud overrides (useful for ZDT migration) + $collectiosOverride = $hooks->trigger('getProjectCollections'); + if(!empty($collectiosOverride)) { + $collections = $collectiosOverride; + } + foreach ($collections as $key => $collection) { if (($collection['$collection'] ?? '') !== Database::METADATA) { continue; diff --git a/src/Appwrite/Hooks/Hooks.php b/src/Appwrite/Hooks/Hooks.php index 00d2f5a9e9..1fe17c7a0e 100644 --- a/src/Appwrite/Hooks/Hooks.php +++ b/src/Appwrite/Hooks/Hooks.php @@ -16,11 +16,12 @@ class Hooks /** * @param mixed[] $params + * @return mixed */ - public function trigger(string $name, array $params = []) + public function trigger(string $name, array $params = []): mixed { if (isset(self::$hooks[$name])) { - call_user_func_array(self::$hooks[$name], $params); + return call_user_func_array(self::$hooks[$name], $params); } } } From 8154b4c131eecf417549bb54916c343075108d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 6 Feb 2024 12:11:27 +0100 Subject: [PATCH 039/406] Linter fix --- app/controllers/api/projects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 135ce9a50f..0f7fad564d 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -187,7 +187,7 @@ App::post('/v1/projects') // Allow Cloud overrides (useful for ZDT migration) $collectiosOverride = $hooks->trigger('getProjectCollections'); - if(!empty($collectiosOverride)) { + if (!empty($collectiosOverride)) { $collections = $collectiosOverride; } From 3a5c233b596676af518f7a668df58710592abcd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 6 Feb 2024 11:47:43 +0000 Subject: [PATCH 040/406] Fix bug --- src/Appwrite/Hooks/Hooks.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Hooks/Hooks.php b/src/Appwrite/Hooks/Hooks.php index 1fe17c7a0e..4840229086 100644 --- a/src/Appwrite/Hooks/Hooks.php +++ b/src/Appwrite/Hooks/Hooks.php @@ -23,5 +23,7 @@ class Hooks if (isset(self::$hooks[$name])) { return call_user_func_array(self::$hooks[$name], $params); } + + return null; } } From a7e95588c6631c298782f51e920e1f8f75191b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 6 Feb 2024 16:08:42 +0100 Subject: [PATCH 041/406] Fix bug on ZDT --- app/controllers/api/projects.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 0f7fad564d..d218750a50 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -185,12 +185,6 @@ App::post('/v1/projects') /** @var array $collections */ $collections = Config::getParam('collections', [])['projects'] ?? []; - // Allow Cloud overrides (useful for ZDT migration) - $collectiosOverride = $hooks->trigger('getProjectCollections'); - if (!empty($collectiosOverride)) { - $collections = $collectiosOverride; - } - foreach ($collections as $key => $collection) { if (($collection['$collection'] ?? '') !== Database::METADATA) { continue; @@ -225,6 +219,8 @@ App::post('/v1/projects') $dbForProject->createCollection($key, $attributes, $indexes); } + $hooks->trigger('afterProjectCreated'); // Useful for ZDT to mirror the project to destination + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($project, Response::MODEL_PROJECT); From 939aec866e17bdd6f5a33e035acc520b3a9f3939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Feb 2024 10:16:17 +0100 Subject: [PATCH 042/406] Add getProxyProjectDatabase hook --- app/controllers/api/projects.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index d218750a50..5318eb4384 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -172,9 +172,14 @@ App::post('/v1/projects') throw new Exception(Exception::PROJECT_ALREADY_EXISTS); } - $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); - $dbForProject->setNamespace("_{$project->getInternalId()}"); - $dbForProject->create(); + // Useful for ZDT to mirror the project to destination + $dbForProject = $hooks->trigger('getProxyProjectDatabase', [ $project, $pools, $cache ]); + + if(empty($dbForProject)) { + $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); + $dbForProject->setNamespace("_{$project->getInternalId()}"); + $dbForProject->create(); + } $audit = new Audit($dbForProject); $audit->setup(); @@ -219,8 +224,6 @@ App::post('/v1/projects') $dbForProject->createCollection($key, $attributes, $indexes); } - $hooks->trigger('afterProjectCreated'); // Useful for ZDT to mirror the project to destination - $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($project, Response::MODEL_PROJECT); From d5216d5b14588b2b99d51d02ae8b6935864371a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Feb 2024 10:17:34 +0100 Subject: [PATCH 043/406] Linter fix --- app/controllers/api/projects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 5318eb4384..4ce673c977 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -175,7 +175,7 @@ App::post('/v1/projects') // Useful for ZDT to mirror the project to destination $dbForProject = $hooks->trigger('getProxyProjectDatabase', [ $project, $pools, $cache ]); - if(empty($dbForProject)) { + if (empty($dbForProject)) { $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); $dbForProject->setNamespace("_{$project->getInternalId()}"); $dbForProject->create(); From a89b73d20bcf489d97b10e3b573d7fa543a770dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Feb 2024 10:48:02 +0100 Subject: [PATCH 044/406] Fix bug --- app/controllers/api/projects.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 4ce673c977..4613070611 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -178,9 +178,10 @@ App::post('/v1/projects') if (empty($dbForProject)) { $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); $dbForProject->setNamespace("_{$project->getInternalId()}"); - $dbForProject->create(); } + $dbForProject->create(); + $audit = new Audit($dbForProject); $audit->setup(); From abb10073853071e28abc006a0f8e2ff0d920af1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Feb 2024 12:04:46 +0100 Subject: [PATCH 045/406] Rework project creation --- app/controllers/api/projects.php | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 4613070611..4fb95cf175 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -172,14 +172,8 @@ App::post('/v1/projects') throw new Exception(Exception::PROJECT_ALREADY_EXISTS); } - // Useful for ZDT to mirror the project to destination - $dbForProject = $hooks->trigger('getProxyProjectDatabase', [ $project, $pools, $cache ]); - - if (empty($dbForProject)) { - $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); - $dbForProject->setNamespace("_{$project->getInternalId()}"); - } - + $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); + $dbForProject->setNamespace("_{$project->getInternalId()}"); $dbForProject->create(); $audit = new Audit($dbForProject); @@ -225,6 +219,8 @@ App::post('/v1/projects') $dbForProject->createCollection($key, $attributes, $indexes); } + $hooks->trigger('afterProjectCreation', [ $project, $pools, $cache ]); + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($project, Response::MODEL_PROJECT); From a0e82ccfab8a6748892734f8d6dfaa44f13dbae1 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Feb 2024 10:35:33 +0200 Subject: [PATCH 046/406] sync with main --- Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1d0d32ed6..a264ac8795 100755 --- a/Dockerfile +++ b/Dockerfile @@ -101,20 +101,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-usage-dump - -# Cloud Executabless -RUN chmod +x /usr/local/bin/hamster && \ - chmod +x /usr/local/bin/volume-sync && \ - chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ - chmod +x /usr/local/bin/patch-recreate-repositories-documents && \ - chmod +x /usr/local/bin/patch-delete-project-collections && \ - chmod +x /usr/local/bin/delete-orphaned-projects && \ - chmod +x /usr/local/bin/clear-card-cache && \ - chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats && \ - chmod +x /usr/local/bin/get-migration-stats && \ - chmod +x /usr/local/bin/create-inf-metric - # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ From 96317f0acb660c483b37666ae4e029ad47fdd48b Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Feb 2024 10:46:24 +0200 Subject: [PATCH 047/406] sync with main --- app/controllers/api/health.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index a85f9da321..e10a0d8b81 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -755,12 +755,12 @@ App::get('/v1/health/queue/failed/:name') Event::MAILS_QUEUE_NAME, Event::FUNCTIONS_QUEUE_NAME, Event::USAGE_QUEUE_NAME, + Event::USAGE_DUMP_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME, Event::CERTIFICATES_QUEUE_NAME, Event::BUILDS_QUEUE_NAME, Event::MESSAGING_QUEUE_NAME, - Event::MIGRATIONS_QUEUE_NAME, - Event::HAMSTER_CLASS_NAME + Event::MIGRATIONS_QUEUE_NAME ]), 'The name of the queue') ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md') From 219b28e9bfdcfd4da9b0e22e3ac2e1646879dc27 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Feb 2024 16:08:40 +0200 Subject: [PATCH 048/406] usage updates --- bin/worker-usage-dump | 3 + docker-compose.yml | 32 ++++++ src/Appwrite/Platform/Workers/Usage.php | 57 +++++++--- src/Appwrite/Platform/Workers/UsageDump.php | 113 ++++++++++++++++++++ src/Appwrite/Platform/Workers/UsageHook.php | 103 ------------------ 5 files changed, 188 insertions(+), 120 deletions(-) create mode 100644 bin/worker-usage-dump create mode 100644 src/Appwrite/Platform/Workers/UsageDump.php delete mode 100644 src/Appwrite/Platform/Workers/UsageHook.php diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump new file mode 100644 index 0000000000..43ca87fcb3 --- /dev/null +++ b/bin/worker-usage-dump @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 395923681d..c4334bb45c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -677,6 +677,38 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + appwrite-worker-usage-dump: + entrypoint: worker-usage-dump + <<: *x-logging + container_name: appwrite-worker-usage-dump + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + + appwrite-schedule: entrypoint: schedule <<: *x-logging diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3809d000f7..49f3344906 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -3,23 +3,23 @@ namespace Appwrite\Platform\Workers; use Exception; +use Utopia\App; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; +use Appwrite\Event\UsageDump; use Utopia\Queue\Message; class Usage extends Action { - protected static array $stats = []; - protected array $periods = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; + private array $stats = []; + private int $lastTriggeredTime = 0; + private int $keys = 0; + private const INFINITY_PERIOD = '_inf_'; + private const KEYS_THRESHOLD = 10000; + - protected const INFINITY_PERIOD = '_inf_'; public static function getName(): string { return 'usage'; @@ -35,26 +35,31 @@ class Usage extends Action ->desc('Usage worker') ->inject('message') ->inject('getProjectDB') - ->callback(function (Message $message, callable $getProjectDB) { - $this->action($message, $getProjectDB); + ->inject('queueForUsageDump') + ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $getProjectDB, $queueForUsageDump); }); + + $this->lastTriggeredTime = time(); } /** * @param Message $message * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB): void + public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } + //Todo Figure out way to preserve keys when the container is being recreated @shimonewman - $payload = $message->getPayload() ?? []; + $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -69,17 +74,35 @@ class Usage extends Action getProjectDB: $getProjectDB ); } - self::$stats[$projectId]['project'] = $project; + + $this->stats[$projectId]['project'] = $project; foreach ($payload['metrics'] ?? [] as $metric) { - if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) { - self::$stats[$projectId]['keys'][$metric['key']] = $metric['value']; + $this->keys++; + if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { + $this->stats[$projectId]['keys'][$metric['key']] = $metric['value']; continue; } - self::$stats[$projectId]['keys'][$metric['key']] += $metric['value']; + + $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; + } + + // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) + if ( + $this->keys >= self::KEYS_THRESHOLD || + (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) + ) { + console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); + + $queueForUsageDump + ->setStats($this->stats) + ->trigger(); + + $this->stats = []; + $this->keys = 0; + $this->lastTriggeredTime = time(); } } - /** * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files. * When we remove a parent document we need to deduct his children aggregation from the project scope. diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php new file mode 100644 index 0000000000..f563578984 --- /dev/null +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -0,0 +1,113 @@ + 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'usage-dump'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + + $this + ->inject('message') + ->inject('getProjectDB') + ->callback(function (Message $message, callable $getProjectDB) { + $this->action($message, $getProjectDB); + }) + ; + } + + /** + * @param Message $message + * @param callable $getProjectDB + * @return void + * @throws Exception + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, callable $getProjectDB): void + { + + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + //Todo rename both usage workers @shimonewman + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + + if ($numberOfKeys === 0) { + continue; + } + + console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); + + try { + $dbForProject = $getProjectDB($project); + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + try { + $dbForProject->createDocument('stats_v2', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => App::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + $dbForProject->decreaseDocumentAttribute( + 'stats_v2', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats_v2', + $id, + 'value', + $value + ); + } + } + } + } + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); + } + } + } +} diff --git a/src/Appwrite/Platform/Workers/UsageHook.php b/src/Appwrite/Platform/Workers/UsageHook.php deleted file mode 100644 index 4781b1e892..0000000000 --- a/src/Appwrite/Platform/Workers/UsageHook.php +++ /dev/null @@ -1,103 +0,0 @@ -setType(Action::TYPE_WORKER_START) - ->inject('register') - ->inject('getProjectDB') - ->callback(function ($register, callable $getProjectDB) { - $this->action($register, $getProjectDB); - }) - ; - } - - /** - * @param $register - * @param $getProjectDB - * @return void - */ - public function action($register, $getProjectDB): void - { - - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000'); - Timer::tick($interval, function () use ($register, $getProjectDB) { - - $offset = count(self::$stats); - $projects = array_slice(self::$stats, 0, $offset, true); - array_splice(self::$stats, 0, $offset); - foreach ($projects as $data) { - $numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0; - $projectInternalId = $data['project']->getInternalId(); - $database = $data['project']['database'] ?? ''; - - console::warning('Ticker started ' . DateTime::now()); - - if ($numberOfKeys === 0) { - continue; - } - - try { - $dbForProject = $getProjectDB($data['project']); - foreach ($data['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats_v2', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => App::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats_v2', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats_v2', - $id, - 'value', - $value - ); - } - } - } - } - } catch (\Exception $e) { - console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); - } - } - }); - } -} From e1e5f5d96b0c3d9dba1c4f93f7914636d83f72bd Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Feb 2024 16:27:01 +0200 Subject: [PATCH 049/406] usage updates --- src/Appwrite/Platform/Services/Workers.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 6573b3124c..22e6dcd56a 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -14,7 +14,7 @@ use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Usage; -use Appwrite\Platform\Workers\UsageHook; +use Appwrite\Platform\Workers\UsageDump; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -33,7 +33,7 @@ class Workers extends Service ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) ->addAction(Hamster::getName(), new Hamster()) - ->addAction(UsageHook::getName(), new UsageHook()) + ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) From 7d38860d16052ce2a144cc332c26d8f92936691c Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Feb 2024 17:04:16 +0200 Subject: [PATCH 050/406] usage updates --- app/worker.php | 15 +++++----- src/Appwrite/Event/Event.php | 3 ++ src/Appwrite/Event/UsageDump.php | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 src/Appwrite/Event/UsageDump.php diff --git a/app/worker.php b/app/worker.php index 1b1f4b9f93..4cf0edbae6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -14,6 +14,7 @@ use Appwrite\Event\Mail; use Appwrite\Event\Migration; use Appwrite\Event\Phone; use Appwrite\Event\Usage; +use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\App; @@ -146,6 +147,9 @@ Server::setResource('log', fn() => new Log()); Server::setResource('queueForUsage', function (Connection $queue) { return new Usage($queue); }, ['queue']); +Server::setResource('queueForUsageDump', function (Connection $queue) { + return new UsageDump($queue); +}, ['queue']); Server::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); @@ -299,12 +303,9 @@ $worker Console::error('[Error] Line: ' . $error->getLine()); }); -try { - $workerStart = $worker->getWorkerStart(); -} catch (\Throwable $error) { - $worker->workerStart(); -} finally { - Console::info("Worker $workerName started"); -} +$worker->workerStart() + ->action(function () use ($workerName) { + Console::info("Worker $workerName started"); + }); $worker->start(); diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index fc12c5b5b3..9f71ef5ebc 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -27,6 +27,9 @@ class Event public const USAGE_QUEUE_NAME = 'v1-usage'; public const USAGE_CLASS_NAME = 'UsageV1'; + public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; + public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; + public const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; public const WEBHOOK_CLASS_NAME = 'WebhooksV1'; diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php new file mode 100644 index 0000000000..8f87908849 --- /dev/null +++ b/src/Appwrite/Event/UsageDump.php @@ -0,0 +1,47 @@ +setQueue(Event::USAGE_DUMP_QUEUE_NAME) + ->setClass(Event::USAGE_DUMP_CLASS_NAME); + } + + /** + * Add Stats. + * + * @param array $stats + * @return self + */ + public function setStats(array $stats): self + { + $this->stats = $stats; + + return $this; + } + + /** + * Sends metrics to the usage worker. + * + * @return string|bool + */ + public function trigger(): string|bool + { + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ + 'stats' => $this->stats, + ]); + } +} From 84e4748625ea43077983f2a1c9ee2c6089a1f7bb Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Feb 2024 17:58:27 +0200 Subject: [PATCH 051/406] usage updates --- .env | 2 +- Dockerfile | 3 ++- src/Appwrite/Platform/Workers/Usage.php | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 31474bf0b6..8a48328f9e 100644 --- a/.env +++ b/.env @@ -77,7 +77,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=60000 +_APP_USAGE_AGGREGATION_INTERVAL=30 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/Dockerfile b/Dockerfile index 9f3ee9b700..5d06d6c828 100755 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-hamster && \ - chmod +x /usr/local/bin/worker-usage + chmod +x /usr/local/bin/worker-usage && \ + chmod +x /usr/local/bin/worker-usage-dump # Cloud Executabless diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 49f3344906..742c404a17 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -46,7 +46,6 @@ class Usage extends Action /** * @param Message $message * @param callable $getProjectDB - * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception From 207811a224e4891c774c890919a1d31d3a3d4e10 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 13 Feb 2024 19:26:23 +0200 Subject: [PATCH 052/406] usage updates --- composer.lock | 83 +++++++++++-------------- src/Appwrite/Platform/Workers/Usage.php | 1 + 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/composer.lock b/composer.lock index 7507243862..ce4c6afca8 100644 --- a/composer.lock +++ b/composer.lock @@ -815,16 +815,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.28.0", + "version": "v1.29.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5" + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/6caa57379c4aec19c0a12a38b59b26487dcfe4b5", - "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", "shasum": "" }, "require": { @@ -832,9 +832,6 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.28-dev" - }, "thanks": { "name": "symfony/polyfill", "url": "https://github.com/symfony/polyfill" @@ -878,7 +875,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0" }, "funding": [ { @@ -894,7 +891,7 @@ "type": "tidelift" } ], - "time": "2023-01-26T09:26:14+00:00" + "time": "2024-01-29T20:11:03+00:00" }, { "name": "utopia-php/abuse", @@ -1190,16 +1187,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.5", + "version": "0.45.6", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6" + "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/0b66a017f817a910acb83e6aea92bccea9571fe6", - "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd", + "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd", "shasum": "" }, "require": { @@ -1240,9 +1237,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.5" + "source": "https://github.com/utopia-php/database/tree/0.45.6" }, - "time": "2024-01-08T17:08:15+00:00" + "time": "2024-02-01T02:33:43+00:00" }, { "name": "utopia-php/domains", @@ -1353,16 +1350,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.1", + "version": "0.33.2", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c" + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/b745607aa1875554a0ad52e28f6db918da1ce11c", - "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c", + "url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", "shasum": "" }, "require": { @@ -1392,9 +1389,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.1" + "source": "https://github.com/utopia-php/http/tree/0.33.2" }, - "time": "2024-01-17T16:48:32+00:00" + "time": "2024-01-31T10:35:59+00:00" }, { "name": "utopia-php/image", @@ -2471,16 +2468,16 @@ }, { "name": "doctrine/deprecations", - "version": "1.1.2", + "version": "1.1.3", "source": { "type": "git", "url": "https://github.com/doctrine/deprecations.git", - "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931" + "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931", - "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab", + "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab", "shasum": "" }, "require": { @@ -2512,9 +2509,9 @@ "homepage": "https://www.doctrine-project.org/", "support": { "issues": "https://github.com/doctrine/deprecations/issues", - "source": "https://github.com/doctrine/deprecations/tree/1.1.2" + "source": "https://github.com/doctrine/deprecations/tree/1.1.3" }, - "time": "2023-09-27T20:04:15+00:00" + "time": "2024-01-30T19:34:25+00:00" }, { "name": "doctrine/instantiator", @@ -4772,16 +4769,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.28.0", + "version": "v1.29.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb" + "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb", - "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4", + "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4", "shasum": "" }, "require": { @@ -4795,9 +4792,6 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.28-dev" - }, "thanks": { "name": "symfony/polyfill", "url": "https://github.com/symfony/polyfill" @@ -4834,7 +4828,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0" }, "funding": [ { @@ -4850,20 +4844,20 @@ "type": "tidelift" } ], - "time": "2023-01-26T09:26:14+00:00" + "time": "2024-01-29T20:11:03+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.28.0", + "version": "v1.29.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "42292d99c55abe617799667f454222c54c60e229" + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229", - "reference": "42292d99c55abe617799667f454222c54c60e229", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec", "shasum": "" }, "require": { @@ -4877,9 +4871,6 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.28-dev" - }, "thanks": { "name": "symfony/polyfill", "url": "https://github.com/symfony/polyfill" @@ -4917,7 +4908,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0" }, "funding": [ { @@ -4933,7 +4924,7 @@ "type": "tidelift" } ], - "time": "2023-07-28T09:04:16+00:00" + "time": "2024-01-29T20:11:03+00:00" }, { "name": "textalk/websocket", @@ -5133,5 +5124,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 742c404a17..49f3344906 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -46,6 +46,7 @@ class Usage extends Action /** * @param Message $message * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception From 62ee10493b66174584eea4a203102f86e5983313 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 15 Feb 2024 10:10:01 +0200 Subject: [PATCH 053/406] usage sms per country code count --- app/init.php | 1 + composer.json | 3 +- composer.lock | 258 +++++++++++++------- src/Appwrite/Event/Usage.php | 1 - src/Appwrite/Platform/Workers/Messaging.php | 13 + 5 files changed, 186 insertions(+), 90 deletions(-) diff --git a/app/init.php b/app/init.php index 80c3670eec..8a407c6eda 100644 --- a/app/init.php +++ b/app/init.php @@ -196,6 +196,7 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length']; const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; const METRIC_MESSAGES = 'messages'; +const METRIC_MESSAGES_COUNTRY_CODE = '{countryCode}.messages'; const METRIC_SESSIONS = 'sessions'; const METRIC_DATABASES = 'databases'; const METRIC_COLLECTIONS = 'collections'; diff --git a/composer.json b/composer.json index 41baf29d9a..2722df51ad 100644 --- a/composer.json +++ b/composer.json @@ -74,7 +74,8 @@ "chillerlan/php-qrcode": "4.3.4", "adhocore/jwt": "1.1.2", "webonyx/graphql-php": "14.11.*", - "league/csv": "9.7.1" + "league/csv": "9.7.1", + "giggsey/libphonenumber-for-php-lite": "8.13.30" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index ca39c7d084..450ecd42fb 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fd03f97115d752d1a94b533ccf570109", + "content-hash": "e9c18e6bda4856800ed1fed582c5766e", "packages": [ { "name": "adhocore/jwt", @@ -402,6 +402,88 @@ ], "time": "2022-09-10T18:51:20+00:00" }, + { + "name": "giggsey/libphonenumber-for-php-lite", + "version": "8.13.30", + "source": { + "type": "git", + "url": "https://github.com/giggsey/libphonenumber-for-php-lite.git", + "reference": "7b205dac7d90a791f7bbae77f2ced2c90b0c5df1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/giggsey/libphonenumber-for-php-lite/zipball/7b205dac7d90a791f7bbae77f2ced2c90b0c5df1", + "reference": "7b205dac7d90a791f7bbae77f2ced2c90b0c5df1", + "shasum": "" + }, + "require": { + "php": "^8.0", + "symfony/polyfill-mbstring": "^1.17" + }, + "conflict": { + "giggsey/libphonenumber-for-php": "*" + }, + "require-dev": { + "ext-dom": "*", + "friendsofphp/php-cs-fixer": "^3.12", + "infection/infection": "^0.26.16", + "pear/pear-core-minimal": "^1.10.11", + "pear/pear_exception": "^1.0.2", + "pear/versioncontrol_git": "^0.5", + "phing/phing": "^2.17.4", + "phpstan/extension-installer": "^1.2", + "phpstan/phpstan": "^1.8", + "phpstan/phpstan-phpunit": "^1.2", + "phpunit/phpunit": "^9.5.26", + "symfony/console": "^6.0" + }, + "suggest": { + "giggsey/libphonenumber-for-php": "Use libphonenumber-for-php for geocoding, carriers, timezones and matching" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "8.x-dev" + } + }, + "autoload": { + "psr-4": { + "libphonenumber\\": "src/" + }, + "exclude-from-classmap": [ + "/src/data/", + "/src/carrier/data/", + "/src/geocoding/data/", + "/src/timezone/data/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Joshua Gigg", + "email": "giggsey@gmail.com", + "homepage": "https://giggsey.com/" + } + ], + "description": "A lite version of giggsey/libphonenumber-for-php, which is a PHP Port of Google's libphonenumber", + "homepage": "https://github.com/giggsey/libphonenumber-for-php-lite", + "keywords": [ + "geocoding", + "geolocation", + "libphonenumber", + "mobile", + "phonenumber", + "validation" + ], + "support": { + "issues": "https://github.com/giggsey/libphonenumber-for-php-lite/issues", + "source": "https://github.com/giggsey/libphonenumber-for-php-lite" + }, + "time": "2024-02-09T12:16:33+00:00" + }, { "name": "jean85/pretty-package-versions", "version": "2.0.5", @@ -813,6 +895,86 @@ ], "time": "2023-03-06T14:43:22+00:00" }, + { + "name": "symfony/polyfill-mbstring", + "version": "v1.29.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-mbstring.git", + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "provide": { + "ext-mbstring": "*" + }, + "suggest": { + "ext-mbstring": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for the Mbstring extension", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "mbstring", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-01-29T20:11:03+00:00" + }, { "name": "symfony/polyfill-php80", "version": "v1.29.0", @@ -2417,16 +2579,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.36.2", + "version": "0.36.3", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece" + "reference": "8d308f7f492545da3e51ea5b91c0778392c40b93" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0aa67479d75f0e0cb7b60454031534d7f0abaece", - "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/8d308f7f492545da3e51ea5b91c0778392c40b93", + "reference": "8d308f7f492545da3e51ea5b91c0778392c40b93", "shasum": "" }, "require": { @@ -2462,9 +2624,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.36.2" + "source": "https://github.com/appwrite/sdk-generator/tree/0.36.3" }, - "time": "2024-01-19T01:04:35+00:00" + "time": "2024-02-14T06:33:38+00:00" }, { "name": "doctrine/deprecations", @@ -4846,86 +5008,6 @@ ], "time": "2024-01-29T20:11:03+00:00" }, - { - "name": "symfony/polyfill-mbstring", - "version": "v1.29.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec", - "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec", - "shasum": "" - }, - "require": { - "php": ">=7.1" - }, - "provide": { - "ext-mbstring": "*" - }, - "suggest": { - "ext-mbstring": "For best performance" - }, - "type": "library", - "extra": { - "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Mbstring\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill for the Mbstring extension", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "mbstring", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2024-01-29T20:11:03+00:00" - }, { "name": "textalk/websocket", "version": "1.5.7", @@ -5124,5 +5206,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index ded276e166..5054a5f0eb 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -59,7 +59,6 @@ class Usage extends Event public function trigger(): string|bool { $client = new Client($this->queue, $this->connection); - return $client->enqueue([ 'project' => $this->getProject(), 'reduce' => $this->reduce, diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 1f3e29c8d5..95c2b4b8de 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -17,6 +17,7 @@ use Utopia\Messaging\Adapters\SMS\Vonage; use Utopia\Platform\Action; use Utopia\Queue\Message; use Appwrite\Event\Usage; +use libphonenumber\PhoneNumberUtil; class Messaging extends Action { @@ -119,6 +120,18 @@ class Messaging extends Action try { $sms->send($message); + $phoneUtil = PhoneNumberUtil::getInstance(); + + try { + $countryCode = $phoneUtil + ->parse($payload['recipient']) + ->getCountryCode(); + + $queueForUsage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_MESSAGES_COUNTRY_CODE), 1); + } catch (\libphonenumber\NumberParseException $e) { + console::error("Error parsing number: " . $e->getMessage()); + } + $queueForUsage ->setProject($project) ->addMetric(METRIC_MESSAGES, 1) From 44ead217ac1c9b9bf6b0e91ed536f75071ea5ed0 Mon Sep 17 00:00:00 2001 From: Shimon Newman Date: Thu, 15 Feb 2024 16:28:02 +0200 Subject: [PATCH 054/406] Update src/Appwrite/Platform/Workers/Messaging.php Co-authored-by: Christy Jacob --- src/Appwrite/Platform/Workers/Messaging.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 95c2b4b8de..cf11077f70 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -129,7 +129,7 @@ class Messaging extends Action $queueForUsage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_MESSAGES_COUNTRY_CODE), 1); } catch (\libphonenumber\NumberParseException $e) { - console::error("Error parsing number: " . $e->getMessage()); + Console::error("Error parsing number: " . $e->getMessage()); } $queueForUsage From ad44c7a430c1dd2426b387cc891dac334ee16893 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 15 Feb 2024 16:43:58 +0200 Subject: [PATCH 055/406] Addressed comments --- src/Appwrite/Platform/Workers/Messaging.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index cf11077f70..4810669cf4 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Exception; +use libphonenumber\NumberParseException; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Document; @@ -128,7 +129,7 @@ class Messaging extends Action ->getCountryCode(); $queueForUsage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_MESSAGES_COUNTRY_CODE), 1); - } catch (\libphonenumber\NumberParseException $e) { + } catch (NumberParseException $e) { Console::error("Error parsing number: " . $e->getMessage()); } From 4e2120952fc135c465c6c2ce0bdd4f50be59815d Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 28 Feb 2024 13:27:36 +0530 Subject: [PATCH 056/406] Fix vcs silent mode --- app/controllers/api/vcs.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 1b0c993e11..0d87c101ad 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -95,7 +95,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $latestCommentId = ''; - if (!empty($providerPullRequestId)) { + if (!empty($providerPullRequestId) && $function->getAttribute('providerSilentMode', false) === false) { $latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('providerPullRequestId', [$providerPullRequestId]), From 064ac2ccf88cb2389c5cc982d2ea477092e49087 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 4 Mar 2024 11:15:45 +0000 Subject: [PATCH 057/406] fix dsn in event test --- tests/unit/Event/EventTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/Event/EventTest.php b/tests/unit/Event/EventTest.php index a430a7fdc6..a84800a28e 100644 --- a/tests/unit/Event/EventTest.php +++ b/tests/unit/Event/EventTest.php @@ -30,7 +30,7 @@ class EventTest extends TestCase $dsn = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); $dsn = explode('=', $dsn); - $dsn = $dsn[0] ?? ''; + $dsn = $dsn[1] ?? ''; $dsn = new DSN($dsn); $connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()); $this->queue = 'v1-tests' . uniqid(); From 86dc85f3a486e221a98e92e27bd4bd6da8f80fe8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 4 Mar 2024 12:17:35 +0000 Subject: [PATCH 058/406] fix file path --- tests/e2e/Services/Databases/DatabasesBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index dbb6d795c6..196199d602 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -4266,7 +4266,7 @@ trait DatabasesBase ], $this->getHeaders()), [ 'documentId' => ID::unique(), 'data' => [ - 'longtext' => file_get_contents('tests/resources/longtext.txt'), + 'longtext' => file_get_contents(__DIR__ . '/../../resources/longtext.txt'), ], 'permissions' => [ Permission::read(Role::user($this->getUser()['$id'])), From 060ec455b3948e6891e601d3e679c41dc39734f2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 4 Mar 2024 12:44:40 +0000 Subject: [PATCH 059/406] fix --- tests/e2e/Services/Locale/LocaleBase.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/Services/Locale/LocaleBase.php b/tests/e2e/Services/Locale/LocaleBase.php index 17b5c66d67..872ec8449f 100644 --- a/tests/e2e/Services/Locale/LocaleBase.php +++ b/tests/e2e/Services/Locale/LocaleBase.php @@ -227,9 +227,9 @@ trait LocaleBase /** * Test for SUCCESS */ - $languages = require('app/config/locale/codes.php'); - $defaultCountries = require('app/config/locale/countries.php'); - $defaultContinents = require('app/config/locale/continents.php'); + $languages = require(__DIR__ . '/../../../../app/config/locale/codes.php'); + $defaultCountries = require(__DIR__ . '/../../../../app/config/locale/countries.php'); + $defaultContinents = require(__DIR__ . '/../../../../app/config/locale/continents.php'); foreach ($languages as $lang) { $response = $this->client->call(Client::METHOD_GET, '/locale/countries', [ From 5f4970befdcb812927c82a7aa41d6d6c4e19704d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 4 Mar 2024 12:49:56 +0000 Subject: [PATCH 060/406] fix path --- tests/e2e/Services/Databases/DatabasesBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 196199d602..79a6c7a329 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -4266,7 +4266,7 @@ trait DatabasesBase ], $this->getHeaders()), [ 'documentId' => ID::unique(), 'data' => [ - 'longtext' => file_get_contents(__DIR__ . '/../../resources/longtext.txt'), + 'longtext' => file_get_contents(__DIR__ . '/../../../extensionsresources/longtext.txt'), ], 'permissions' => [ Permission::read(Role::user($this->getUser()['$id'])), From 5e6f53d6aa6fabd46b26e9e3cc1f527155b22259 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 4 Mar 2024 12:50:49 +0000 Subject: [PATCH 061/406] reclaim pool --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 9696b08f6c..3cb9ffb2ad 100644 --- a/app/init.php +++ b/app/init.php @@ -1161,7 +1161,7 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { ->setMetadata('host', \gethostname()) ->setMetadata('project', 'console') ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - + $pools->get('console')->reclaim(); return $database; }, ['pools', 'cache']); From 57d7995e3653a89e26128d3017b8a40588048b8b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 4 Mar 2024 12:58:28 +0000 Subject: [PATCH 062/406] fix file path --- .../Services/Projects/ProjectsConsoleClientTest.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 574ff7dcc5..869e0c3582 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -798,7 +798,7 @@ class ProjectsConsoleClientTest extends Scope public function testUpdateProjectOAuth($data): array { $id = $data['projectId'] ?? ''; - $providers = require('app/config/providers.php'); + $providers = require(__DIR__ . '/../../../../app/config/providers.php'); /** * Test for SUCCESS @@ -909,7 +909,7 @@ class ProjectsConsoleClientTest extends Scope public function testUpdateProjectAuthStatus($data): array { $id = $data['projectId'] ?? ''; - $auth = require('app/config/auth.php'); + $auth = require(__DIR__ . '/../../../../app/config/auth.php'); $originalEmail = uniqid() . 'user@localhost.test'; $originalPassword = 'password'; @@ -1732,7 +1732,7 @@ class ProjectsConsoleClientTest extends Scope $this->assertNotEmpty($project['body']['$id']); $id = $project['body']['$id']; - $services = require('app/config/services.php'); + $services = require(__DIR__ . '/../../../../app/config/services.php'); /** * Test for Disabled @@ -1806,7 +1806,7 @@ class ProjectsConsoleClientTest extends Scope { $id = $data['projectId']; - $services = require('app/config/services.php'); + $services = require(__DIR__ . '/../../../../app/config/services.php'); /** * Test for Disabled @@ -1880,7 +1880,7 @@ class ProjectsConsoleClientTest extends Scope { $id = $data['projectId']; - $services = require('app/config/services.php'); + $services = require(__DIR__ . '/../../../../app/config/services.php'); /** * Test for Disabled From 78e4c0611bb1237f09f6ac1ead3e391457bc5851 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 4 Mar 2024 20:02:45 +0100 Subject: [PATCH 063/406] Update deps --- composer.json | 4 +- composer.lock | 120 ++++++++++++++++++++++++++++---------------------- 2 files changed, 70 insertions(+), 54 deletions(-) diff --git a/composer.json b/composer.json index da5bee6dfb..90e8e7ec20 100644 --- a/composer.json +++ b/composer.json @@ -46,11 +46,11 @@ "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.34.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.35.*", + "utopia-php/audit": "0.36.*", "utopia-php/cache": "0.9.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.45.*", + "utopia-php/database": "dev-feat-0.45-isolation-modes as 0.46.0", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index c789878e99..3d511d9d0f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f7a4173bb5adeea3518f949fa883a282", + "content-hash": "ea035b22fabb5e8518ee5fd6a6afbc3a", "packages": [ { "name": "adhocore/jwt", @@ -1187,16 +1187,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.6", + "version": "dev-feat-0.45-isolation-modes", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd" + "reference": "086c18e03d2bb3d1b12d1ccfc45d47da1473e05f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd", - "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd", + "url": "https://api.github.com/repos/utopia-php/database/zipball/086c18e03d2bb3d1b12d1ccfc45d47da1473e05f", + "reference": "086c18e03d2bb3d1b12d1ccfc45d47da1473e05f", "shasum": "" }, "require": { @@ -1204,7 +1204,7 @@ "ext-pdo": "*", "php": ">=8.0", "utopia-php/cache": "0.9.*", - "utopia-php/framework": "0.*.*", + "utopia-php/framework": "0.33.*", "utopia-php/mongo": "0.3.*" }, "require-dev": { @@ -1237,9 +1237,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.6" + "source": "https://github.com/utopia-php/database/tree/feat-0.45-isolation-modes" }, - "time": "2024-02-01T02:33:43+00:00" + "time": "2024-03-04T18:38:13+00:00" }, { "name": "utopia-php/domains", @@ -2822,20 +2822,21 @@ }, { "name": "phar-io/manifest", - "version": "2.0.3", + "version": "2.0.4", "source": { "type": "git", "url": "https://github.com/phar-io/manifest.git", - "reference": "97803eca37d319dfa7826cc2437fc020857acb53" + "reference": "54750ef60c58e43759730615a392c31c80e23176" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/manifest/zipball/97803eca37d319dfa7826cc2437fc020857acb53", - "reference": "97803eca37d319dfa7826cc2437fc020857acb53", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/54750ef60c58e43759730615a392c31c80e23176", + "reference": "54750ef60c58e43759730615a392c31c80e23176", "shasum": "" }, "require": { "ext-dom": "*", + "ext-libxml": "*", "ext-phar": "*", "ext-xmlwriter": "*", "phar-io/version": "^3.0.1", @@ -2876,9 +2877,15 @@ "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", "support": { "issues": "https://github.com/phar-io/manifest/issues", - "source": "https://github.com/phar-io/manifest/tree/2.0.3" + "source": "https://github.com/phar-io/manifest/tree/2.0.4" }, - "time": "2021-07-20T11:28:43+00:00" + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2024-03-03T12:33:53+00:00" }, { "name": "phar-io/version", @@ -3217,16 +3224,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.30", + "version": "9.2.31", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089" + "reference": "48c34b5d8d983006bd2adc2d0de92963b9155965" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/ca2bd87d2f9215904682a9cb9bb37dda98e76089", - "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/48c34b5d8d983006bd2adc2d0de92963b9155965", + "reference": "48c34b5d8d983006bd2adc2d0de92963b9155965", "shasum": "" }, "require": { @@ -3283,7 +3290,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.30" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.31" }, "funding": [ { @@ -3291,7 +3298,7 @@ "type": "github" } ], - "time": "2023-12-22T06:47:57+00:00" + "time": "2024-03-02T06:37:42+00:00" }, { "name": "phpunit/php-file-iterator", @@ -3689,16 +3696,16 @@ }, { "name": "sebastian/cli-parser", - "version": "1.0.1", + "version": "1.0.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/cli-parser.git", - "reference": "442e7c7e687e42adc03470c7b668bc4b2402c0b2" + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/442e7c7e687e42adc03470c7b668bc4b2402c0b2", - "reference": "442e7c7e687e42adc03470c7b668bc4b2402c0b2", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/2b56bea83a09de3ac06bb18b92f068e60cc6f50b", + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b", "shasum": "" }, "require": { @@ -3733,7 +3740,7 @@ "homepage": "https://github.com/sebastianbergmann/cli-parser", "support": { "issues": "https://github.com/sebastianbergmann/cli-parser/issues", - "source": "https://github.com/sebastianbergmann/cli-parser/tree/1.0.1" + "source": "https://github.com/sebastianbergmann/cli-parser/tree/1.0.2" }, "funding": [ { @@ -3741,7 +3748,7 @@ "type": "github" } ], - "time": "2020-09-28T06:08:49+00:00" + "time": "2024-03-02T06:27:43+00:00" }, { "name": "sebastian/code-unit", @@ -3987,16 +3994,16 @@ }, { "name": "sebastian/diff", - "version": "4.0.5", + "version": "4.0.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131" + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/74be17022044ebaaecfdf0c5cd504fc9cd5a7131", - "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/ba01945089c3a293b01ba9badc29ad55b106b0bc", + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc", "shasum": "" }, "require": { @@ -4041,7 +4048,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/diff/issues", - "source": "https://github.com/sebastianbergmann/diff/tree/4.0.5" + "source": "https://github.com/sebastianbergmann/diff/tree/4.0.6" }, "funding": [ { @@ -4049,7 +4056,7 @@ "type": "github" } ], - "time": "2023-05-07T05:35:17+00:00" + "time": "2024-03-02T06:30:58+00:00" }, { "name": "sebastian/environment", @@ -4116,16 +4123,16 @@ }, { "name": "sebastian/exporter", - "version": "4.0.5", + "version": "4.0.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "ac230ed27f0f98f597c8a2b6eb7ac563af5e5b9d" + "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/ac230ed27f0f98f597c8a2b6eb7ac563af5e5b9d", - "reference": "ac230ed27f0f98f597c8a2b6eb7ac563af5e5b9d", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/78c00df8f170e02473b682df15bfcdacc3d32d72", + "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72", "shasum": "" }, "require": { @@ -4181,7 +4188,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/exporter/issues", - "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.5" + "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.6" }, "funding": [ { @@ -4189,20 +4196,20 @@ "type": "github" } ], - "time": "2022-09-14T06:03:37+00:00" + "time": "2024-03-02T06:33:00+00:00" }, { "name": "sebastian/global-state", - "version": "5.0.6", + "version": "5.0.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "bde739e7565280bda77be70044ac1047bc007e34" + "reference": "bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bde739e7565280bda77be70044ac1047bc007e34", - "reference": "bde739e7565280bda77be70044ac1047bc007e34", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9", + "reference": "bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9", "shasum": "" }, "require": { @@ -4245,7 +4252,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.6" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.7" }, "funding": [ { @@ -4253,7 +4260,7 @@ "type": "github" } ], - "time": "2023-08-02T09:26:13+00:00" + "time": "2024-03-02T06:35:11+00:00" }, { "name": "sebastian/lines-of-code", @@ -4973,16 +4980,16 @@ }, { "name": "theseer/tokenizer", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/theseer/tokenizer.git", - "reference": "b2ad5003ca10d4ee50a12da31de12a5774ba6b96" + "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b2ad5003ca10d4ee50a12da31de12a5774ba6b96", - "reference": "b2ad5003ca10d4ee50a12da31de12a5774ba6b96", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", + "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", "shasum": "" }, "require": { @@ -5011,7 +5018,7 @@ "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", "support": { "issues": "https://github.com/theseer/tokenizer/issues", - "source": "https://github.com/theseer/tokenizer/tree/1.2.2" + "source": "https://github.com/theseer/tokenizer/tree/1.2.3" }, "funding": [ { @@ -5019,7 +5026,7 @@ "type": "github" } ], - "time": "2023-11-20T00:12:19+00:00" + "time": "2024-03-03T12:36:25+00:00" }, { "name": "twig/twig", @@ -5094,9 +5101,18 @@ "time": "2023-11-21T18:54:41+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-0.45-isolation-modes", + "alias": "0.46.0", + "alias_normalized": "0.46.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From fcc5a747d12a86863302f711f5ac9ec58db0fc7d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 4 Mar 2024 20:26:11 +0100 Subject: [PATCH 064/406] Fix test --- tests/e2e/General/HTTPTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index bf8f6de279..fd5cf7e925 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -31,7 +31,7 @@ class HTTPTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); $this->assertEquals('Appwrite', $response['headers']['server']); $this->assertEquals('GET, POST, PUT, PATCH, DELETE', $response['headers']['access-control-allow-methods']); - $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']); + $this->assertEquals('Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-Appwrite-Timeout, X-Appwrite-Share-Tables, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma, X-Fallback-Cookies', $response['headers']['access-control-allow-headers']); $this->assertEquals('X-Fallback-Cookies', $response['headers']['access-control-expose-headers']); $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']); $this->assertEquals('true', $response['headers']['access-control-allow-credentials']); From c768375e130f231dce8e24d0eaf5d19d1d4118b5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 4 Mar 2024 20:54:35 +0100 Subject: [PATCH 065/406] Set tenant for CLI scope db --- app/cli.php | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/app/cli.php b/app/cli.php index e3f0ba72d7..c98cc62e7c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -103,7 +103,19 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; - $database->setNamespace('_' . $project->getInternalId()); + + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + $database + ->setShareTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace(''); + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + return $database; } @@ -116,6 +128,18 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databases[$databaseName] = $database; + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + $database + ->setShareTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace(''); + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + $database ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) From 7fc214afbef4a77ccd9819138c81349eb1d67d5c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 5 Mar 2024 11:55:29 +0100 Subject: [PATCH 066/406] Update libs --- composer.json | 2 +- composer.lock | 55 +++++++++++++++++++++------------------------------ 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/composer.json b/composer.json index 90e8e7ec20..33d3b158ec 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "utopia-php/cache": "0.9.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-0.45-isolation-modes as 0.46.0", + "utopia-php/database": "0.45.7", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 3d511d9d0f..c2150e47dc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ea035b22fabb5e8518ee5fd6a6afbc3a", + "content-hash": "9ab28ccc7cde35e834aac175d331c346", "packages": [ { "name": "adhocore/jwt", @@ -895,23 +895,23 @@ }, { "name": "utopia-php/abuse", - "version": "0.34.0", + "version": "0.34.1", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "49f84abc0f2317ba5b55af809cf6b411253c4855" + "reference": "c8497650cd3ee0d04fe8a9e6391572262c1d89e0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/49f84abc0f2317ba5b55af809cf6b411253c4855", - "reference": "49f84abc0f2317ba5b55af809cf6b411253c4855", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/c8497650cd3ee0d04fe8a9e6391572262c1d89e0", + "reference": "c8497650cd3ee0d04fe8a9e6391572262c1d89e0", "shasum": "" }, "require": { "ext-curl": "*", "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.46.*" + "utopia-php/database": "0.45.*" }, "require-dev": { "laravel/pint": "1.5.*", @@ -938,9 +938,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.34.0" + "source": "https://github.com/utopia-php/abuse/tree/0.34.1" }, - "time": "2023-12-08T17:04:15+00:00" + "time": "2024-03-05T10:43:41+00:00" }, { "name": "utopia-php/analytics", @@ -990,21 +990,21 @@ }, { "name": "utopia-php/audit", - "version": "0.36.0", + "version": "0.36.1", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "b490f06d687fc1510d44ddce305afb81462ddd62" + "reference": "9db48fed0558c33c9ddbb493fe2171844a3f0fd4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/b490f06d687fc1510d44ddce305afb81462ddd62", - "reference": "b490f06d687fc1510d44ddce305afb81462ddd62", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/9db48fed0558c33c9ddbb493fe2171844a3f0fd4", + "reference": "9db48fed0558c33c9ddbb493fe2171844a3f0fd4", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.46.*" + "utopia-php/database": "0.45.*" }, "require-dev": { "laravel/pint": "1.5.*", @@ -1031,9 +1031,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.36.0" + "source": "https://github.com/utopia-php/audit/tree/0.36.1" }, - "time": "2023-12-08T17:04:53+00:00" + "time": "2024-03-05T10:50:30+00:00" }, { "name": "utopia-php/cache", @@ -1187,16 +1187,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-0.45-isolation-modes", + "version": "0.45.7", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "086c18e03d2bb3d1b12d1ccfc45d47da1473e05f" + "reference": "f508c5fcec8e4b2c323a12dd4355a8cb0cc6ad03" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/086c18e03d2bb3d1b12d1ccfc45d47da1473e05f", - "reference": "086c18e03d2bb3d1b12d1ccfc45d47da1473e05f", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f508c5fcec8e4b2c323a12dd4355a8cb0cc6ad03", + "reference": "f508c5fcec8e4b2c323a12dd4355a8cb0cc6ad03", "shasum": "" }, "require": { @@ -1237,9 +1237,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-0.45-isolation-modes" + "source": "https://github.com/utopia-php/database/tree/0.45.7" }, - "time": "2024-03-04T18:38:13+00:00" + "time": "2024-03-05T10:28:02+00:00" }, { "name": "utopia-php/domains", @@ -5101,18 +5101,9 @@ "time": "2023-11-21T18:54:41+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-0.45-isolation-modes", - "alias": "0.46.0", - "alias_normalized": "0.46.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 1b7ce8fc61af94f3bc24a86b50ceb3c775f7fc32 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 6 Mar 2024 10:59:05 +0100 Subject: [PATCH 067/406] Debug --- app/cli.php | 2 ++ app/controllers/api/projects.php | 3 +++ app/init.php | 2 ++ app/worker.php | 3 +++ 4 files changed, 10 insertions(+) diff --git a/app/cli.php b/app/cli.php index c98cc62e7c..324d1c948c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -105,6 +105,7 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database = $databases[$databaseName]; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -129,6 +130,7 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databases[$databaseName] = $database; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 552c3f786b..06f219fb6f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -158,6 +158,8 @@ App::post('/v1/projects') $database = DATABASE_SHARED_TABLES; } + \var_dump('DATABASE: ' . $database); + try { $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, @@ -198,6 +200,7 @@ App::post('/v1/projects') $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); if ($database === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $dbForProject ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/app/init.php b/app/init.php index 06d508e27a..705b8173e4 100644 --- a/app/init.php +++ b/app/init.php @@ -1136,6 +1136,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -1184,6 +1185,7 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/app/worker.php b/app/worker.php index 989f1223af..207762fef9 100644 --- a/app/worker.php +++ b/app/worker.php @@ -76,6 +76,7 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setMetadata('project', $project->getId()); if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -115,6 +116,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $database = $databases[$databaseName]; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -139,6 +141,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $databases[$databaseName] = $database; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) From 5b00737b55a81dd015b9a38c33ad4e5755281fc8 Mon Sep 17 00:00:00 2001 From: Shimon Newman Date: Wed, 6 Mar 2024 18:09:13 +0100 Subject: [PATCH 068/406] Update app/init.php Co-authored-by: Eldad A. Fux --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 8a407c6eda..ebd790b065 100644 --- a/app/init.php +++ b/app/init.php @@ -196,7 +196,7 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length']; const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; const METRIC_MESSAGES = 'messages'; -const METRIC_MESSAGES_COUNTRY_CODE = '{countryCode}.messages'; +const METRIC_MESSAGES_COUNTRY = '{countryCode}.messages'; const METRIC_SESSIONS = 'sessions'; const METRIC_DATABASES = 'databases'; const METRIC_COLLECTIONS = 'collections'; From 3312ed1c5da4b60ce9721a636a02dab182e4ef8d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 6 Mar 2024 21:14:55 +0100 Subject: [PATCH 069/406] Debug --- app/cli.php | 2 - app/init.php | 16 +- app/worker.php | 3 - composer.lock | 12 +- names2.json | 1527 ++++++++++++++++++++++++++++++++++++++++++++++++ test.php | 0 6 files changed, 1540 insertions(+), 20 deletions(-) create mode 100644 names2.json create mode 100644 test.php diff --git a/app/cli.php b/app/cli.php index 324d1c948c..c98cc62e7c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -105,7 +105,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database = $databases[$databaseName]; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -130,7 +129,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databases[$databaseName] = $database; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/app/init.php b/app/init.php index 705b8173e4..2493386390 100644 --- a/app/init.php +++ b/app/init.php @@ -1135,18 +1135,17 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); + //if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $database - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } return $database; }, ['pools', 'dbForConsole', 'cache', 'project']); @@ -1185,7 +1184,6 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/app/worker.php b/app/worker.php index 207762fef9..989f1223af 100644 --- a/app/worker.php +++ b/app/worker.php @@ -76,7 +76,6 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setMetadata('project', $project->getId()); if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -116,7 +115,6 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $database = $databases[$databaseName]; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) @@ -141,7 +139,6 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $databases[$databaseName] = $database; if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); $database ->setShareTables(true) ->setTenant($project->getInternalId()) diff --git a/composer.lock b/composer.lock index c2150e47dc..905abf7103 100644 --- a/composer.lock +++ b/composer.lock @@ -2764,16 +2764,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.0.1", + "version": "v5.0.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "2218c2252c874a4624ab2f613d86ac32d227bc69" + "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/2218c2252c874a4624ab2f613d86ac32d227bc69", - "reference": "2218c2252c874a4624ab2f613d86ac32d227bc69", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/139676794dc1e9231bf7bcd123cfc0c99182cb13", + "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13", "shasum": "" }, "require": { @@ -2816,9 +2816,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.2" }, - "time": "2024-02-21T19:24:10+00:00" + "time": "2024-03-05T20:51:40+00:00" }, { "name": "phar-io/manifest", diff --git a/names2.json b/names2.json new file mode 100644 index 0000000000..a32b5ca8d5 --- /dev/null +++ b/names2.json @@ -0,0 +1,1527 @@ +{ + "album-database": 1, + "default": 7, + "Renew-Tech-DB": 1, + "posts": 7, + "vreddb": 1, + "test-db": 12, + "friendly-secret": 1, + "recipeDB": 2, + "TestDb": 2, + "first": 5, + "test": 96, + "Securelee": 1, + "cpldb": 1, + "SpeakingDB": 1, + "StoriesDB": 1, + "UserSignup": 1, + "WrittenDB": 1, + "db1": 17, + "CourcesDB": 1, + "Learnhub": 1, + "Users": 17, + "notes": 3, + "main": 28, + "Event planning ": 1, + "SupAut": 1, + "userTable": 1, + "coredb": 1, + "event-planner": 1, + "P01": 1, + "chatgpt-database": 1, + "JQQ Memes": 1, + "nocodb": 1, + "TaskList": 1, + "Retail": 1, + "Test": 30, + "Subscriptions": 1, + "Primary DB": 1, + "travelappdatabase": 1, + "DB": 7, + "back-end": 1, + "mydb": 5, + "Chats": 1, + "secrets": 1, + "pine": 1, + "MamaPut": 1, + "simpleApp": 1, + "gossip": 1, + "ecom_database": 1, + "edge-data": 1, + "vizualGitDb": 1, + "ticket": 1, + "toDo": 1, + "beacon": 1, + "hero": 1, + "users": 33, + "Twitter Test": 21, + "code-share": 1, + "DbName": 1, + "MyRecipePad": 1, + "twitter_clone": 3, + "event_db": 1, + "timers": 1, + "parsiukai": 1, + "AuthDatabase": 2, + "pfs_dev": 1, + "contratista": 1, + "support-local": 1, + "hey": 1, + "mindmemoDB": 1, + "E commerse": 1, + "my_basketball_club": 1, + "Amaad": 1, + "testdb": 21, + "twitter-clone": 7, + "srikalyanam": 1, + "todos": 27, + "Recipes": 2, + "todo": 18, + "demo": 12, + "confessus": 1, + "game": 3, + "primary": 2, + "dummy workout": 1, + "FitPulse": 1, + "vender": 1, + "workouts": 1, + "fresh": 1, + "hello": 3, + "auth": 1, + "domain": 1, + "Glitch": 1, + "Teste": 1, + "UsersData": 1, + "Linda": 1, + "supabse": 1, + "events": 4, + "dopecut-data": 1, + "lll": 1, + "FurrBuddiesDB": 1, + "VeterinarianDB": 1, + "test-sso-akhilesh": 1, + "cdv": 1, + "testing": 4, + "USDZ models": 1, + "book1": 1, + "Events": 4, + "cmpt-int-db": 1, + "yupio_db": 1, + "content": 3, + "test1": 2, + "okr db": 1, + "ckh-sport-equipment": 1, + "Twitter test": 1, + "Companies": 1, + "datas": 1, + "first-database": 1, + "tiger": 1, + "evo_db": 2, + "ProHacks": 1, + "cards": 1, + "projects": 3, + "app": 29, + "conversations": 2, + "Logging": 1, + "images": 2, + "podbase": 1, + "MYMATCH": 1, + "TestDB1": 2, + "m": 1, + "Tasks": 3, + "Recipes-Tracker": 1, + "db": 21, + "restaurants-details": 1, + "TEST": 2, + "first-db": 1, + "product-info-management": 1, + "demodb": 1, + "chats": 1, + "project-db": 2, + "logs_db": 1, + "users_db": 2, + "hustlehub-db": 1, + "profile": 1, + "jha": 1, + "photosharing": 1, + "mongi": 1, + "tupropiedad": 1, + "spotify database": 1, + "usersWatchlists": 1, + "watchlistsDB": 1, + "twitter_test": 1, + "chatDB": 1, + "development": 4, + "11": 1, + "vm": 1, + "manga mania db": 1, + "tutorialDB": 1, + "quizapp": 3, + "GitSafari": 1, + "lulucat": 1, + "testdatabase1101": 1, + "shops": 1, + "Salt-user-db": 1, + "insta_clone": 1, + "jobo": 1, + "vectors": 1, + "prueba": 2, + "Tech Blog": 1, + "socio_db": 1, + "Eventhub": 1, + "we-code": 1, + "jalauncodinghub": 1, + "beebeer_db": 1, + "recipes": 3, + "Twitter Clone dev": 1, + "testDb": 1, + "Rizq Sharing": 1, + "abc": 1, + "4_db": 1, + "dtdb": 1, + "clientUser": 1, + "asnad database": 1, + "ded": 1, + "yesusa_portal": 1, + "guidepilot-db": 1, + "hackathon_entries": 1, + "Dime": 1, + "blogs": 8, + "Genx": 1, + "SQD": 1, + "hippoDatabase": 1, + "Todo": 2, + "merchdjv_productdb": 1, + "sample": 2, + "datatest": 1, + "esp": 1, + "project": 2, + "letsdesign": 1, + "Studentpos": 1, + "movies": 2, + "quickstart": 1, + "Backup": 1, + "Default": 11, + "XXX": 4, + "vue_crud": 1, + "lessons": 1, + "test-database": 2, + "HellDB": 1, + "myDB": 1, + "products": 5, + "Straddle": 1, + "reketupa": 1, + "Restore": 1, + "Setup": 1, + "task-db": 1, + "money_manager": 1, + "3tee_databas": 1, + "jobs": 2, + "multi-resume": 1, + "my_db": 1, + "Trobia": 1, + "Movies": 5, + "foo": 2, + "Event Planner": 1, + "bleed": 1, + "DB1": 3, + "database": 9, + "fitadom": 1, + "hashnode": 1, + "stenodb": 1, + "nexus": 1, + "hosts": 1, + "space_launches_info": 1, + "dev": 9, + "testbase": 2, + "tinto": 1, + "test_db": 5, + "Data info": 1, + "demo db": 1, + "Studucatrial": 1, + "twitterClone": 2, + "Babu": 1, + "Cara": 1, + "booking": 2, + "Url Shortener": 1, + "votify": 1, + "lynaesh_dashboard": 1, + "URLShort": 1, + "SocialMedia": 1, + "appDatabase": 10, + "Database 1": 1, + "ps-innovation-day": 1, + "AppDatabase": 161, + "appdatabase": 1, + "svellodatabase": 1, + "user": 5, + "TrelloDatabase": 2, + "first_app": 1, + "boats": 1, + "intratec-tecnologia": 1, + "chat-app": 1, + "DnD database": 1, + "appwrite": 2, + "admin_users_database": 1, + "app database": 2, + "234234": 1, + "dusers": 1, + "trello-database": 4, + "TaskerDB": 1, + "meddb": 1, + "dsuite-db": 2, + "fleekytrendsProducts": 1, + "healthservices": 1, + "openchat": 1, + "Aurelia DB": 1, + "Locations": 1, + "AppDB": 3, + "VideoCall": 1, + "Development": 2, + "TrelloApp": 1, + "type-frenzy": 1, + "config": 1, + "FakeDB": 1, + "dev_01": 1, + "register": 1, + "gratitude_jar_db": 1, + "priceers": 1, + "Huntdb": 1, + "TrelloDb": 2, + "prasad": 1, + "brtbhrthtrhhtrhtrnh": 1, + "CodingFlux Database": 1, + "Flair Database": 1, + "FVB": 1, + "Admin": 2, + "emailusers": 1, + "ITrackemail": 1, + "notificationtoken": 1, + "orgcode": 1, + "schoolreq": 1, + "Triperbus": 1, + "vehicles": 1, + "ATWS Databases": 1, + "Buildrankers-Prod": 1, + "Discussion Forum": 1, + "CanbanDatabase": 1, + "chipIn MVP database": 1, + "jamshaidtaskifyDatabase": 1, + "school": 2, + "todo-db": 2, + "LearningApp": 1, + "twitter test": 2, + "TestDB": 9, + "appdata": 1, + "flutter_appwrite_col": 1, + "flutter_appwrite": 1, + "PromanageDatabase": 1, + "landing": 1, + "bank-data": 1, + "moni": 1, + "StuffDatabase": 1, + "alertapp-prod": 1, + "alertapp-qa": 1, + "Walldb": 1, + "role:on": 1, + "leaderboardlearn": 1, + "Taskify": 1, + "DB-PAYMENT-PAGE": 1, + "ggg": 2, + "TrelloCloneDB": 1, + "kanban-gpt-app": 1, + "awc-news": 1, + "data": 6, + "orders": 1, + "ding": 1, + "bisky_chat": 1, + "vaibgram-db": 1, + "trello-clone": 3, + "appdb": 3, + "trelo": 1, + "tpmt": 1, + "languageEx": 1, + "virtualBot": 1, + "Database test": 1, + "Main": 9, + "rnd": 1, + "trips": 1, + "ProductDatabase": 1, + "atesting": 1, + "cakebudget": 1, + "oya_db": 1, + "Raven": 1, + "TaskAppDatabase": 1, + "main_db": 1, + "exams": 1, + "Notes": 1, + "root": 2, + "Exercises": 1, + "estudos": 1, + "store": 2, + "trello-db": 1, + "ProdAmigoDB": 1, + "chikigram": 1, + "Viagens": 1, + "brainwars_db": 1, + "app-database": 1, + "Trello": 2, + "Admins": 1, + "wax": 1, + "undanganonline": 1, + "Sunarp": 1, + "recipto_db": 1, + "trello_clone_db": 1, + "dB": 2, + "Messages": 3, + "Dadabase": 1, + "SecurePass": 1, + "Registrations": 1, + "Sponsors": 1, + "Trello Database": 2, + "contact_form": 1, + "chat_db": 1, + "app_database": 1, + "watchlist": 1, + "myorg": 1, + "temir": 1, + "Todos": 4, + "Website Color CSS": 1, + "testonio": 1, + "teste": 3, + "trello-v2": 1, + "tikDb": 1, + "Queries": 1, + "shophub": 1, + "appDatabases": 1, + "trelloDatabase": 1, + "appDataBase": 1, + "mmm": 1, + "workdiary": 1, + "Specs99 DB": 1, + "App database": 1, + "Frank": 1, + "app_data": 1, + "app_pop_up_db": 1, + "express_pay_app_database": 1, + "nextcloudify_db_dev": 1, + "main database": 1, + "VitaPlate": 1, + "writepost": 1, + "trial": 1, + "BataapDB": 1, + "collectify_database": 1, + "pso2_items_database": 1, + "agis": 1, + "twitter": 4, + "HabitProDB": 1, + "TripArchitectDB": 1, + "airiedb": 1, + "purupdb": 1, + "flutter": 7, + "flutter_database": 12, + "geoteams": 1, + "Ecom": 1, + "Trabajadores": 1, + "HnosUceda": 1, + "etracker": 2, + "Reviews": 1, + "trello-clone-db": 1, + "AllPos": 1, + "liqlid-mvp-main": 1, + "db01": 2, + "Trello dev": 1, + "child": 1, + "shortcuts": 1, + "TrelloCloneAppDatabase": 1, + "leagueFusion": 1, + "activity_1000k": 1, + "activity_100k": 1, + "TaskManager-Projects": 1, + "SocialMediaAppDB": 1, + "devdactic": 2, + "dbtian": 1, + "BlogMd": 1, + "Trello_Clone_AppDatabase": 1, + "waste-collection": 1, + "user_details": 1, + "appwrite_hackathon": 1, + "new": 4, + "messages": 6, + "Render2023": 1, + "testinggg": 1, + "chatwrite": 3, + "Chatty": 1, + "information": 1, + "chattingDB": 1, + "NoteDb": 1, + "app-db": 3, + "123": 1, + "webook": 1, + "apt": 1, + "User": 6, + "hackerspace": 1, + "general": 5, + "ChatDB": 2, + "renderatl-2023": 1, + "General": 1, + "zoro": 1, + "testData": 1, + "ns-db": 1, + "tbUsuario": 1, + "trails": 1, + "GameRefill Shop": 1, + "fitdb": 1, + "cccmedia": 1, + "robodoradca": 1, + "astro": 1, + "369": 1, + "likakuDB": 1, + "clinick-db": 1, + "devices": 2, + "maindb": 1, + "notebooks": 1, + "chatting-application": 1, + "Storage": 1, + "CSE_DB": 1, + "AppTodo": 1, + "rideonparts": 1, + "appwritedb": 1, + "Testdb": 1, + "wisetally": 1, + "staging": 3, + "taskkify-db": 1, + "Ellume360": 1, + "EllumeLogin": 1, + "trell-app-database": 1, + "products database": 1, + "AInews": 1, + "Session": 1, + "riia-db": 1, + "backfillJobs": 1, + "Settings": 1, + "Log": 1, + "App": 2, + "Document": 1, + "oneDeal": 1, + "scan_and_go_db": 1, + "goraiya db": 1, + "CombiDataBase": 1, + "dcg": 1, + "todoData": 1, + "notestester": 1, + "recipe": 1, + "mqtt_auth": 1, + "pumprecord-db": 1, + "Slideshub": 1, + "Prescriptions ": 1, + "AppDataBase": 2, + "business_Aid": 1, + "trello_app": 1, + "slack-app-db": 1, + "taskapp": 1, + "trello-app": 1, + "SorageX": 1, + "superdb": 1, + "Osd": 1, + "trivia_project": 1, + "Emp": 1, + "userdetails": 1, + "HRIS": 1, + "Account": 1, + "tetris": 1, + "task-ai_database": 1, + "SheroDatabase": 1, + "UserData": 2, + "DashboardDb": 1, + "mydatabase": 1, + "event name, description, date, and location": 1, + "Payments": 1, + "Skarban": 1, + "VGPT": 1, + "WhatsappDatabase": 1, + "meetings": 1, + "Icicle_Statistics": 1, + "CRUX": 1, + "crud": 1, + "hosting": 1, + "Tt": 1, + "tables": 1, + "arbah": 1, + "globalbulls": 1, + "calligrapy": 1, + "instant_messenger": 1, + "Finance": 1, + "cook it up": 1, + "main-db": 1, + "IRA": 1, + "AppDtabase": 1, + "who fed the pet data": 1, + "agenda": 1, + "invoices": 2, + "notifications": 2, + "patient": 1, + "userSettings": 1, + "nmn": 1, + "upinvoice": 1, + "base64": 1, + "swingfire-app": 1, + "photos": 3, + "meetify": 1, + "gptmanaDB": 1, + "pos": 2, + "school_council": 1, + "62d985ab68e0e4b60293": 1, + "62dd7c6bbe54693b8a32": 1, + "secure-chat": 1, + "android-test": 1, + "commpcs_pro": 1, + "TEST DATABASE": 1, + "TaskWalletDB": 1, + "Contracts": 1, + "sampleDB": 1, + "telegram_bot": 1, + "AZED": 1, + "Hapix-db": 1, + "prod": 4, + "App-db": 1, + "quotes": 1, + "Dummy data": 1, + "User appointments": 1, + "chrono": 1, + "VarsityWorldDevelopment": 1, + "ServiceProviders": 1, + "r": 1, + "Murad": 1, + "ScheduleHero": 4, + "testindb": 1, + "flutter_db": 1, + "defaultDb": 1, + "History": 1, + "Distributors-Labels": 1, + "LoggingDB": 1, + "reservas": 1, + "SurveyDatabase": 1, + "no-bullshit-recipes-database": 1, + "lvr_db": 1, + "db1234": 1, + "myClass": 1, + "trello": 2, + "mrElayaDb": 1, + "memos": 1, + "Accounts": 1, + "Commissions": 1, + "lines": 1, + "BKB": 2, + "Elements": 2, + "design_resources": 1, + "public_data": 1, + "users_data": 1, + "X-connect": 1, + "inventory": 1, + "OrganizerDB": 1, + "AppDatabasexxxx": 1, + "animals": 1, + "transport": 1, + "markly": 1, + "db_app": 1, + "board": 2, + "ahaporn": 1, + "subscribers": 1, + "databases": 1, + "message": 1, + "vote": 1, + "portfolio": 2, + "chat": 12, + "QuestList": 1, + "Personal Life Assistant DB": 1, + "C Programming Club": 1, + "groupevents-db": 1, + "resumix": 1, + ",": 1, + "Negotiations": 1, + "Basedate": 4, + "twitterdb": 1, + "STO": 1, + "ai-generator-db-v1": 1, + "e-commerce": 1, + "MADG7": 1, + "TrelloAppDatabase": 1, + "firstdb": 1, + "Items": 1, + "digitalsign": 1, + "e-commerce-app": 1, + "appDb": 2, + "anycasts": 1, + "food": 2, + "noodles-db": 1, + "properly": 1, + "repulse": 1, + "geolocation1_db": 1, + "Updated Database": 9, + "db_kharcha_aamdani": 1, + "Jajanan Lobet": 1, + "NodeToDo": 1, + "mutual_funds": 1, + "nextjs": 1, + "bybit": 1, + "medlogs": 1, + "life-loop": 1, + "ll": 1, + "client": 1, + "AllThumbs": 1, + "Test DB": 2, + "todoApp": 1, + "t.db1": 1, + "dog_sitting": 1, + "todosss": 1, + "doovu": 1, + "tasks": 4, + "tasks_list": 7, + "cagnotte": 1, + "6482f68757302733e335": 1, + "64d7c796b08f76ec51f6": 1, + "db_demo": 1, + "bus": 1, + "election-app": 1, + "a": 2, + "employees": 2, + "database_name_m001": 1, + "resume": 2, + "ImmerseToDo": 1, + "[PROD] krello db": 1, + "myMlmDb": 1, + "ct": 1, + "RacketScoreDB": 1, + "khataApp": 1, + "luna": 1, + "Abc": 1, + "TestDatabase1": 1, + "dev12": 1, + "taskies main database": 1, + "SO-SKILLED": 1, + "hp01": 17, + "iku": 1, + "items": 2, + "mi-tunes-db": 1, + "TaskifyDatabase": 1, + "Ttatar": 1, + "myDatabase": 2, + "TaskBro": 1, + "appslause_trello_db": 1, + "blogposts": 18, + "trundling": 1, + "base_database": 1, + "bookListTest": 1, + "PU-Chd": 1, + "Pu-Chd": 1, + "postapi": 1, + "FamProDB": 1, + "photoshare": 1, + "Blog": 4, + "W.Task-AppDatabase": 1, + "TrelloAppDataBase": 1, + "fleet": 1, + "reservation": 1, + "drb": 1, + "DemoDatabase": 1, + "storage alpha": 1, + "contentus": 1, + "contentus-shadow-db": 1, + "whatsapp": 1, + "explore-aw-db": 1, + "phone auth demo": 1, + "readlits-db": 1, + "facemodel": 1, + "categories": 1, + "pms": 1, + "mohali-id": 1, + "UserDatabase": 1, + "collabverse": 1, + "gratitude-db": 1, + "my-digital-manager-database": 1, + "budget": 1, + "jmdb": 1, + "okaki-sensoren": 1, + "Personality": 1, + "UserDB": 1, + "ghgh": 1, + "JinjaCovers": 1, + "Blogs": 1, + "ihmtransact": 1, + "MoodMapDB": 1, + "pruebaadz": 1, + "cv": 1, + "fff": 1, + "monedaDB": 1, + "Main Database": 1, + "thenonpp_dev": 1, + "product": 3, + "ecommerce": 1, + "TASKS": 1, + "portals": 1, + "twitterTest": 1, + "sample_data": 1, + "Phalam": 1, + "journalDB": 1, + "LeaveFormStore": 1, + "Event DB": 1, + "Employees": 1, + "transactions_db": 1, + "melange": 1, + "myskoolDb": 1, + "blood_bank_db": 1, + "blood_chat_db": 1, + "bracket_assets_db": 1, + "bracket_db": 1, + "bracket_invitations_db": 1, + "chats_db": 1, + "counselling_appointments_db": 1, + "counselling_db": 1, + "discussion_assets_db": 1, + "discussion_db": 1, + "discussion_talks_db": 1, + "discussion_user_stats": 1, + "events_db": 1, + "members_db": 1, + "user_notification_db": 1, + "seats": 1, + "A": 2, + "Marks": 1, + "d1": 1, + "Diary Dev": 1, + "Load Test": 1, + "wer": 1, + "Products": 3, + "medical": 1, + "xanwas": 1, + "committable-backend": 1, + "Food": 1, + "Management": 1, + "chef_mani": 1, + "DB Billetterie": 1, + "Appwrite DB": 1, + "mio-admin": 1, + "User Data": 2, + "News DB": 1, + "info_Sossis": 1, + "DB_AULAS": 1, + "todo-final": 1, + "Evd": 1, + "players": 1, + "HungerZero": 1, + "Twitter db": 1, + "connectkoreaair": 1, + "s": 1, + "xpenzave_db": 1, + "students": 1, + "chatBlog": 1, + "newChatBlog": 1, + "TEAMING PORTAL CHAT ": 1, + "rss_db": 1, + "stella": 1, + "Database": 3, + "Twitter": 2, + "TrelloDB": 2, + "lorgger": 1, + "mindly_db": 1, + "Testdatabase": 1, + "soundboard": 1, + "GroupChatDB": 1, + "one": 2, + "Heatank": 1, + "bucket_data": 1, + "user_data": 1, + "primary_db": 1, + "Project": 1, + "tripbot": 1, + "sell_smart_main": 1, + "mainDB": 1, + "hackathon": 1, + "AppDatabse": 1, + "MyApp": 2, + "bookstore": 1, + "devverse-db": 1, + "expense-care-db": 1, + "pins": 1, + "expensasaurus": 1, + "Ireland Made": 1, + "sac": 1, + "WriterBlog": 1, + "snippad-main": 1, + "Trello-data": 1, + "silai": 1, + "codecollab": 1, + "Transactions": 1, + "Local Log": 1, + "AppDatabases": 1, + "FirstTestDB": 1, + "SignalClone": 1, + "FileStore": 1, + "21yards": 1, + "todo-app": 2, + "Insudash": 1, + "zokoboss": 1, + "Karte": 2, + "Rel-TEST": 1, + "olimpodb": 1, + "zinduo flashcard": 1, + "newDB": 1, + "EventScape": 1, + "Konsulting": 1, + "Demo": 1, + "redmatters": 1, + "essai_db": 1, + "my-stirixis-database": 1, + "MyBlogs": 1, + "blogpost": 3, + "zlist_db": 1, + "movieData": 1, + "Players": 1, + "mindful-me": 1, + "lolo": 1, + "sharebitedb": 1, + "master": 1, + "sf": 1, + "noteenc": 1, + "blogposts ": 1, + "Blog-Posts": 1, + "db-1": 1, + "testDB": 2, + "641f01fb3f6a1bc22c7e": 1, + "mysql": 1, + "HabitDB": 1, + "agents": 1, + "Tam Duc Apparel DB": 1, + "blog-post": 1, + "stores": 1, + "department": 1, + "sop": 1, + "Distributa": 3, + "rnxcode_db": 1, + "netflix_clone": 1, + "guizapp": 1, + "BlogPosts": 1, + "Chat": 2, + "Bedevelopers-Database": 1, + "likumbi": 1, + "OptionSuperManDB": 1, + "article": 1, + "NewDB": 1, + "MyGameStudio": 1, + "default_db": 1, + "userdata": 2, + "Main DB": 1, + "travfddb": 1, + "expenses": 1, + "incomes": 1, + "consultations_database": 1, + "doctors_database": 1, + "patients_database": 1, + "dealfuel": 1, + "Zenivote": 1, + "Portfolio Details": 1, + "Abhidhan Rajendra Kosh": 1, + "common": 1, + "AppUser": 1, + "studygang": 1, + "Peakfit.Db": 1, + "Credentials": 1, + "destinations": 1, + "fun2print_cc": 1, + "billing1": 1, + "studygangdb": 1, + "autana_ide": 1, + "betadev": 1, + "Vibey": 1, + "Slam Book": 1, + "TeleGPT": 1, + "Clips": 1, + "Quizmaster": 1, + "Office Hours DB": 1, + "canva order": 1, + "VideoWatching": 1, + "DataBase": 1, + "sharif-invoice": 1, + "Hack Questions": 1, + "er": 1, + "course": 1, + "Almost Netflix Project": 1, + "Database_sree": 1, + "Quizapp": 1, + "UNIT": 1, + "Public": 2, + "SalesT": 1, + "ZIC": 1, + "Employee": 1, + "Pets": 1, + "twitter-database": 2, + "dev-db": 1, + "track-db": 1, + "demo-db": 2, + "test db": 2, + "teste_db": 1, + "db_lumes": 1, + "SuperWalk": 1, + "rbac2": 1, + "Animesh": 1, + "Bereal": 1, + "PdfTest": 1, + "colonium-users": 1, + "dbMyPrice": 1, + "Production": 2, + "admin_db": 1, + "report-feelings": 1, + "browser": 1, + "protasker": 1, + "shop": 1, + "trigger-tech": 1, + "user data": 1, + "online_env": 1, + "DC3Mobi": 1, + "webappdb": 1, + "synapp": 1, + "pages": 1, + "collective-tales-dev": 1, + "Bildwurf DB": 1, + "Cards": 1, + "ok": 1, + "tmp": 1, + "StandUp": 1, + "Web Database": 1, + "TodoDB": 1, + "Prueba": 1, + "pafunn": 1, + "profiles": 2, + "nice-support": 1, + "pulse": 1, + "jargonjar": 1, + "xsx": 1, + "event-lly": 1, + "Sprittie": 1, + "test_database": 1, + "StroyOko": 1, + "Sparkle": 1, + "Coworkers": 1, + "Canva Templates": 1, + "production": 5, + "home-management": 1, + "dt": 1, + "vodo_db": 1, + "MyTestDB": 1, + "LinkedIn DB": 1, + "blog": 15, + "Example Database": 1, + "companion_db": 1, + "uni_sort": 1, + "rfg_app": 1, + "twitter-clone-project": 1, + "Restaurant_profiles": 1, + "trytowin": 1, + "db_test": 2, + "gallery": 1, + "usersData": 1, + "core": 1, + "IM": 1, + "TechBase": 1, + "ANDON": 1, + "awtest-db": 1, + "Widgets": 1, + "ember": 1, + "asdasdasd": 1, + "courrierflex": 1, + "TunezStream": 1, + "PmanDatabase": 1, + "weintegral": 1, + "TrelloData": 1, + "Test db": 1, + "bandom": 1, + "rezoom-eh-db": 1, + "WordJournalDB": 1, + "99": 1, + "T": 1, + "dop_exams": 1, + "database1": 3, + "asx": 1, + "myilsc": 1, + "Student Base": 1, + "shortify": 1, + "Cheelz Database": 1, + "TESLA Web Database": 1, + "Shorely Ibiza": 1, + "test-server": 1, + "Local_Gems": 1, + "articles": 1, + "AppwriteDb": 1, + "Auction": 1, + "lol": 1, + "todowrite-db": 1, + "BooksDB": 1, + "LoanDB": 2, + "dali": 1, + "JCI app": 1, + "OMS": 1, + "PIM": 1, + "RFx": 1, + "Data": 6, + "acctg1": 1, + "hris1": 1, + "hris2": 1, + "JobTitle": 2, + "CRUD database": 1, + "Foodtracker": 1, + "typical_food": 1, + "demoDB": 1, + "base1": 1, + "trello database": 1, + "demo-db-2": 1, + "Hypertrophy": 1, + "Dojo database": 1, + "skinvasai": 1, + "sgveti": 1, + "medicademy-db": 1, + "habbit": 1, + "Task": 1, + "Data ": 1, + "mvp": 1, + "wwwg": 1, + "public-system": 1, + "student_data": 1, + "chatting": 1, + "Kharidoit": 1, + "kj": 1, + "local": 1, + "wallpaper": 1, + "workspace": 1, + "sarcom-admin": 1, + "tickets": 1, + "ifiye-db": 1, + "Dev": 1, + "Prod": 2, + "TooodooosDB": 1, + "Climast database ": 1, + "My Provisioned DB": 1, + "logs": 2, + "Ascended": 1, + "avx": 1, + "geoloyaltydb": 1, + "yulidb": 1, + "public": 2, + "cms": 1, + "shop together": 1, + "UsersPosts": 1, + "tetst1": 1, + "blogdatabase": 1, + "Twitter test db": 1, + "vhhfdh": 1, + "baket": 1, + "i2e_sw_account": 1, + "New": 1, + "appwrite-test": 1, + "example database": 1, + "storyTable": 1, + "Clothing": 1, + "SV dataflow": 1, + "hgfdjd": 1, + "CoorfyDB": 1, + "purchased": 1, + "Countdown": 1, + "moe": 1, + "rawabialkharef": 1, + "123database": 1, + "shareamDB": 1, + "twitter_db": 1, + "fluttiyomi": 1, + "trends": 1, + "servers": 1, + "Oblongata": 1, + "Cygnus": 1, + "appwriteLearningDb": 1, + "Product": 1, + "PedroDB": 1, + "opos": 1, + "top-team-opos": 1, + "trainings": 1, + "converstations": 1, + "Modern Deals": 1, + "WebStorage": 1, + "apiTestOnline": 1, + "rajmade": 1, + "Profile image url": 1, + "Demo 1": 1, + "Akordi": 1, + "syncms": 1, + "pranav": 1, + "Oracle ": 1, + "dashboard": 1, + "deti": 1, + "exms": 1, + "rm-test": 1, + "test2": 1, + "Grids": 1, + "ProductionDatabase": 1, + "TestDatabase": 1, + "Yukt": 1, + "z": 1, + "CMS": 1, + "Entries": 1, + "books": 1, + "ewoerp": 1, + "AVImark Reports": 1, + "customtabs": 1, + "Database-1": 1, + "sampledatabase": 1, + "Deptos": 1, + "Sugam's Database": 1, + "Test-DB": 2, + "sms": 1, + "skyone-todo": 1, + "tiktok-clone": 1, + "YouTube Videos": 1, + "automata": 1, + "radium": 1, + "TestDb1": 1, + "etherdb": 1, + "global_db": 1, + "near.db": 1, + "Yggdrasil": 1, + "comentarios_db": 1, + "photo_fusion": 1, + "seashipfinder": 1, + "issues": 1, + "DojoDB": 1, + "jkjk": 1, + "dms": 1, + "online_blood_and_organ_donation_system": 1, + "reminders": 1, + "some-database": 1, + "socialPixer": 1, + "home-cafe": 1, + "betrendz-dev": 1, + "snapgram": 3, + "donation-retriever": 1, + "storeDB": 1, + "befree-users": 1, + "Chemistry stuff": 1, + "Core Database": 1, + "instaFaceGram": 1, + "theUiratec": 1, + "blendr": 1, + "occulo": 1, + "nextdb": 1, + "routes": 1, + "user-data": 1, + "JDB": 1, + "shorelyIbizaProduction": 1, + "socialApp": 1, + "SimplGram": 1, + "Snapgram_Data": 1, + "snapcy": 1, + "Image_DB": 1, + "hello_world": 1, + "download": 1, + "Mydb": 2, + "trip": 1, + "oral": 1, + "dddd": 1, + "shopping-list": 1, + "omnibrain": 1, + "basic": 1, + "unity": 1, + "xerox_bot": 1, + "first_db": 1, + "SIEL DB": 1, + "db-preprod": 1, + "testme": 2, + "dreamdb": 1, + "anochat-database": 1, + "anochat-db": 2, + "anochat-dn": 1, + "sovo_db": 1, + "pumpkin": 1, + "marketplace-test": 1, + "PG": 1, + "user list": 1, + "userdb": 1, + "amri": 1, + "members": 1, + "daily-match": 1, + "gdt-members": 1, + "Systems": 1, + "swiftform": 1, + "fanaka": 1, + "dbDev": 1, + "my-first-db": 1, + "Vehicles": 1, + "Kennels": 1, + "calendula": 1, + "asdasd": 1, + "tsproject": 1, + "mallas": 1, + "variedades": 1, + "qs": 1, + "asd": 1, + "mdc_magazine": 1, + "mdc_store": 1, + "School Me DB": 1, + "jczd": 1, + "kupczyk": 1, + "Testing": 1, + "sprintcap": 1, + "CloudShared": 1, + "urls": 1, + "willy": 1, + "My first database": 1, + "Links": 1, + "Twitter Clone": 1, + "functions": 1, + "viewCollections": 1, + "stagging": 1, + "Payment Records": 1, + "AED307Group14": 1, + "rubrica": 1, + "histronia": 1, + "td": 1, + "doctor": 1, + "features": 1, + "park": 1, + "College Cutoff": 1, + "like": 1, + "todos-db": 1, + "TESTDB": 1, + "1": 2, + "BetterChatGPT": 1, + "FIrst db": 1, + "bot_database_test": 1, + "cloud-maturity": 1, + "vendas": 1, + "swirly test": 1, + "Terst": 1, + "CC_db": 1, + "pos db": 1, + "Groups": 1, + "testdatabase": 1, + "kaze": 1, + "Connect": 1, + "NUV-AzTechs-Database": 1, + "Payment Gateway": 1, + "testdata": 2, + "tradeassistant": 1, + "mediation_database": 1, + "dbTest": 1, + "basketball": 1, + "belanja_db": 1, + "Fun": 1, + "mytest": 1, + "FirstDatabase": 1, + "administration": 1, + "consumers": 1, + "mregistryDatabase": 1, + "Twst": 1, + "blogdb": 1, + "search_me": 1, + "aide": 1, + "plantebyt": 1, + "kittens": 1, + "waters": 1, + "aaaaa": 1, + "altrec_db": 1, + "atai": 1, + "Home": 1, + "swifty": 1, + "schedule": 1, + "andimar": 1, + "Manlou": 1, + "cuitanDB": 1, + "smartbcard": 1, + "afe": 1, + "Openstartup": 1, + "Stock": 1, + "drivers": 1, + "sample_order": 1, + "5870": 1, + "kjhg": 1, + "xva": 1, + "MyTemplates": 1, + "Fiesta Data": 1, + "patients": 1, + "animated_dev": 1, + "classapp-db": 1, + "dbb": 1, + "twilio": 1, + "ads": 1, + "Saved Images": 1, + "Name": 1, + "MyDB": 2, + "RipplesCode": 1, + "LeaguesWorld": 1, + "outlet": 1, + "scorer": 1, + "itistime": 1, + "Manuk Biru DB": 1, + "guests": 1, + "questions": 1, + "AI Poker": 1, + "nash_app": 1, + "cars": 1, + "ccc": 1, + "Bookings": 1, + "proj": 1, + "Twitter Clone Database": 1, + "quantasip": 1, + "navigation": 1, + "check": 1, + "testovacka_db": 1, + "base": 1, + "Testbase": 1, + "CoDeveloper Database": 1, + "sample_db": 1, + "dg_1142_dbstore": 1, + "todo-fawp-db": 1, + "project_database": 1, + "Issues": 1, + "cinehousedb": 1, + "foodItems": 1, + "Snowcamp": 1, + "codeApi": 1, + "debify": 1, + "Ymir": 1, + "NetflixDB": 1, + "Database2": 1, + "brainTeaser": 1, + "Fableit Dev": 1, + "Twitter Database": 1, + "Users ": 1, + "portal": 1, + "svelte-test": 1, + "TatiaDb": 1, + "FirstDB": 1, + "galaxies": 1, + "cf_db": 1, + "formData": 1, + "Letter of Credit DB": 1, + "Attribute": 1, + "Inventar": 1, + "banco": 1, + "Announce": 1, + "Maintenance": 1, + "aptkr": 1, + "streamlinks": 1, + "maze1": 1, + "Macellpower": 1, + "Student": 1, + "Forms": 1, + "company-management": 1, + "DBDFM1": 1, + "skool": 1, + "foods": 1, + "Harsh": 1, + "loginator-db": 1, + "Blubbidb": 1, + "Parchin_DB": 1, + "Fitness_App": 1, + "Books": 1, + "usersDB": 1, + "yum": 1, + "yumShares": 12, + "bottledrive": 1, + "website": 1, + "ampPanda": 1, + "TestData": 1, + "oppenheimer": 1, + "sampleDatabase": 1, + "ims_db": 1, + "dingleberry": 1, + "shopaholic-db": 1, + "Sample Database": 1, + "delta-co": 1, + "myLabDB": 1, + "posts database": 1, + "Newsletter": 1, + "order_management": 1, + "psn-kadi-svelte": 1, + "PeetFeederDb": 1, + "yyui": 1, + "taskninja": 1, + "hmnz-core": 1, + "payroll": 1, + "Demo_DB": 1, + "reco-teams-alpha": 1, + "foodly": 1, + "set": 1, + "tst": 1, + "HuntPile": 1, + "-": 1, + "Images": 1, + "money": 1, + "relax": 1, + "todo2": 1, + "vmfk": 1, + "ident": 1, + "recommend": 1, + "Cinemagix ": 1, + "picoLens": 1, + "Organizations": 1, + "safescan": 1, + "first db": 1, + "Photos": 1, + "nn": 1, + "todo's": 1, + "db_news": 1, + "listings": 1, + "to-do-list": 1, + "event-manager": 1, + "iconinstallation": 1, + "charshow_db": 1, + "snipdb": 1, + "socio-flare": 1, + "venicedb": 1, + "drip": 1, + "playground": 1, + "shelters-db": 1, + "kudoscrate": 1, + "Task_management_app": 1, + "ptp": 2, + "workout-history": 1, + "Muslim": 1, + "asklocaldb": 1, + "databiatch": 1, + "QuizProDB": 1, + "discussions": 1, + "ics": 1, + "UsersDetails": 1, + "my-chat": 1, + "easy-prompt-test": 1, + "cmt4": 1, + "cmt5": 1, + "cmt6": 1, + "atglancecronconfig": 1, + "atglancenewsdb": 1, + "mainbase": 1, + "quiz": 2, + "Let's Connect": 1, + "Bill Pecker": 1, + "Database1": 1, + "prompt-store": 1, + "DB-test-1": 1, + "SevyController": 1, + "buzz2xdb": 1, + "recipeYardDatabase": 1, + "quickd": 1, + "Chat Database": 1, + "mods": 1, + "opslog": 1, + "live_feedback": 1, + "stoker_database": 1, + "Education": 1, + "No Signal": 1, + "Appwrite CMS": 1, + "firstDatabase": 1, + "LinkedIn": 1, + "neptune_blindbox": 1, + "Msgs": 1, + "table": 1, + "post": 1, + "rareBeauty": 1, + "formsDB": 1 +} \ No newline at end of file diff --git a/test.php b/test.php new file mode 100644 index 0000000000..e69de29bb2 From 2bed17de35bcd3a7f2ea3e958b8b67dd32f89079 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 6 Mar 2024 21:50:31 +0100 Subject: [PATCH 070/406] Debug --- app/cli.php | 28 ++++++++++++++-------------- app/init.php | 14 +++++++------- app/worker.php | 42 +++++++++++++++++++++--------------------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/app/cli.php b/app/cli.php index c98cc62e7c..76e15e2ff9 100644 --- a/app/cli.php +++ b/app/cli.php @@ -104,17 +104,17 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; - if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { +// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $database - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } return $database; } @@ -128,17 +128,17 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databases[$databaseName] = $database; - if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { +// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $database - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } $database ->setNamespace('_' . $project->getInternalId()) diff --git a/app/init.php b/app/init.php index 2493386390..26ebdab7a9 100644 --- a/app/init.php +++ b/app/init.php @@ -1183,17 +1183,17 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { +// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $database - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } }); if (isset($databases[$databaseName])) { diff --git a/app/worker.php b/app/worker.php index 989f1223af..a3ff06f9a5 100644 --- a/app/worker.php +++ b/app/worker.php @@ -75,17 +75,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()); - if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { +// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $database - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } return $database; }, ['cache', 'register', 'message', 'dbForConsole']); @@ -114,17 +114,17 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; - if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { +// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $database - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } return $database; } @@ -138,17 +138,17 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $databases[$databaseName] = $database; - if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { +// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $database - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } return $database; }; From 1ffb80f2f79e511718b62518b3715e7f34d16664 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Mar 2024 10:37:57 +0100 Subject: [PATCH 071/406] Fix realtime db --- app/realtime.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index f7fc7070a4..4f0ba634e7 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -78,10 +78,17 @@ function getProjectDB(Document $project): Database $database = new Database($dbAdapter, getCache()); - $database - ->setNamespace('_' . $project->getInternalId()) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()); +// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + $database + ->setShareTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace(''); +// } else { +// $database +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } return $database; } From 1db9d8b9aba110286d3151f149ab0c44a3424d3c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Mar 2024 10:40:03 +0100 Subject: [PATCH 072/406] Fix method refs --- app/cli.php | 1 - app/controllers/api/projects.php | 15 +++++++-------- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 2 +- src/Appwrite/Platform/Workers/Hamster.php | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/app/cli.php b/app/cli.php index 76e15e2ff9..0780ee5e5a 100644 --- a/app/cli.php +++ b/app/cli.php @@ -141,7 +141,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, // } $database - ->setNamespace('_' . $project->getInternalId()) ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 06f219fb6f..20f8f16c11 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -199,18 +199,17 @@ App::post('/v1/projects') $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); - if ($database === DATABASE_SHARED_TABLES) { - \var_dump('Using shared tables'); +// if ($database === DATABASE_SHARED_TABLES) { $dbForProject ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); - } else { - $dbForProject - ->setShareTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } +// } else { +// $dbForProject +// ->setShareTables(false) +// ->setTenant(null) +// ->setNamespace('_' . $project->getInternalId()); +// } $dbForProject->create(); diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index b76e0428d7..a8d88e1459 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -102,7 +102,7 @@ class GetMigrationStats extends Action ->getResource(); $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); /** Get Project ID */ diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php index 0fb705d0f7..98bc56ee14 100644 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -122,7 +122,7 @@ class Hamster extends Action ->getResource(); $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $statsPerProject = []; From 503fb2ca82290e6ce8e01c44ad687212e411c162 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Mar 2024 12:49:54 +0100 Subject: [PATCH 073/406] Debug --- app/init.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 26ebdab7a9..a20598397d 100644 --- a/app/init.php +++ b/app/init.php @@ -1146,8 +1146,8 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, // ->setTenant(null) // ->setNamespace('_' . $project->getInternalId()); // } - - return $database; + return null; +// return $database; }, ['pools', 'dbForConsole', 'cache', 'project']); App::setResource('dbForConsole', function (Group $pools, Cache $cache) { From b857fa3cdc690112e52d87bf3d3fa1ccdf74d363 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Mar 2024 14:06:33 +0100 Subject: [PATCH 074/406] Debug --- app/init.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index a20598397d..b5ebd4bb9f 100644 --- a/app/init.php +++ b/app/init.php @@ -1210,7 +1210,8 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database = new Database($dbAdapter, $cache); $databases[$databaseName] = $database; $configure($database); - return $database; + return null; +// return $database; }; }, ['pools', 'dbForConsole', 'cache']); From 8a73203635f781599361984a34b28c5fd2823679 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Mar 2024 14:52:13 +0100 Subject: [PATCH 075/406] Revert debugs --- app/cli.php | 28 ++++++------- app/controllers/api/projects.php | 16 +++---- app/init.php | 36 ++++++++-------- app/realtime.php | 14 +++---- app/worker.php | 42 +++++++++---------- .../Platform/Tasks/DeleteOrphanedProjects.php | 17 ++++++-- .../Platform/Tasks/GetMigrationStats.php | 17 ++++++-- src/Appwrite/Platform/Workers/Hamster.php | 17 ++++++-- 8 files changed, 110 insertions(+), 77 deletions(-) diff --git a/app/cli.php b/app/cli.php index 0780ee5e5a..e246101e1c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -104,17 +104,17 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; -// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } return $database; } @@ -128,17 +128,17 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databases[$databaseName] = $database; -// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } $database ->setMetadata('host', \gethostname()) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 20f8f16c11..89ca9e15fe 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -130,7 +130,7 @@ App::post('/v1/projects') } $databaseOverride = App::getEnv('_APP_DATABASE_OVERRIDE'); - $index = array_search($databaseOverride, $databases); + $index = \array_search($databaseOverride, $databases); if ($index !== false) { $database = $databases[$index]; } else { @@ -199,17 +199,17 @@ App::post('/v1/projects') $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); -// if ($database === DATABASE_SHARED_TABLES) { + if ($database === DATABASE_SHARED_TABLES) { $dbForProject ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $dbForProject -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $dbForProject + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } $dbForProject->create(); diff --git a/app/init.php b/app/init.php index b5ebd4bb9f..a7ffcc5154 100644 --- a/app/init.php +++ b/app/init.php @@ -1135,19 +1135,19 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - //if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } - return null; -// return $database; + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; }, ['pools', 'dbForConsole', 'cache', 'project']); App::setResource('dbForConsole', function (Group $pools, Cache $cache) { @@ -1183,17 +1183,17 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); -// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } }); if (isset($databases[$databaseName])) { @@ -1210,8 +1210,8 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database = new Database($dbAdapter, $cache); $databases[$databaseName] = $database; $configure($database); - return null; -// return $database; + + return $database; }; }, ['pools', 'dbForConsole', 'cache']); diff --git a/app/realtime.php b/app/realtime.php index 4f0ba634e7..80c5a4fbaf 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -78,17 +78,17 @@ function getProjectDB(Document $project): Database $database = new Database($dbAdapter, getCache()); -// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } return $database; } diff --git a/app/worker.php b/app/worker.php index a3ff06f9a5..989f1223af 100644 --- a/app/worker.php +++ b/app/worker.php @@ -75,17 +75,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()); -// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } return $database; }, ['cache', 'register', 'message', 'dbForConsole']); @@ -114,17 +114,17 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if (isset($databases[$databaseName])) { $database = $databases[$databaseName]; -// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } return $database; } @@ -138,17 +138,17 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso $databases[$databaseName] = $database; -// if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { + if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database ->setShareTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); -// } else { -// $database -// ->setShareTables(false) -// ->setTenant(null) -// ->setNamespace('_' . $project->getInternalId()); -// } + } else { + $database + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } return $database; }; diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 860cc3a8a2..5fe4b10472 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -83,15 +83,26 @@ class DeleteOrphanedProjects extends Action } try { - $db = $project->getAttribute('database'); + $database = $project->getAttribute('database'); $adapter = $pools - ->get($db) + ->get($database) ->pop() ->getResource(); $dbForProject = new Database($adapter, $cache); $dbForProject->setDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); + + if ($database === DATABASE_SHARED_TABLES) { + $dbForProject + ->setShareTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace(''); + } else { + $dbForProject + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } $collectionsCreated = 0; $cnt++; diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index a8d88e1459..fa47c27c6d 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -95,15 +95,26 @@ class GetMigrationStats extends Action Console::info("Getting stats for {$project->getId()}"); try { - $db = $project->getAttribute('database'); + $database = $project->getAttribute('database'); $adapter = $pools - ->get($db) + ->get($database) ->pop() ->getResource(); $dbForProject = new Database($adapter, $cache); $dbForProject->setDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); + + if ($database === DATABASE_SHARED_TABLES) { + $dbForProject + ->setShareTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace(''); + } else { + $dbForProject + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } /** Get Project ID */ $stats['Project ID'] = $project->getId(); diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php index 98bc56ee14..167e4d7fe7 100644 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -115,15 +115,26 @@ class Hamster extends Action Console::log("Getting stats for Project {$project->getId()}"); try { - $db = $project->getAttribute('database'); + $database = $project->getAttribute('database'); $adapter = $pools - ->get($db) + ->get($database) ->pop() ->getResource(); $dbForProject = new Database($adapter, $cache); $dbForProject->setDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); + + if ($database === DATABASE_SHARED_TABLES) { + $dbForProject + ->setShareTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace(''); + } else { + $dbForProject + ->setShareTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } $statsPerProject = []; From f166a1c20114c891d42f2f519f5e00bc63e26099 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Mar 2024 17:43:44 +0100 Subject: [PATCH 076/406] Update db --- composer.json | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 33d3b158ec..e5aef1ef6d 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "utopia-php/cache": "0.9.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.45.7", + "utopia-php/database": "0.45.8", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 905abf7103..b0046320f9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9ab28ccc7cde35e834aac175d331c346", + "content-hash": "767fe773a75b04ff6c22eac8b2c1498b", "packages": [ { "name": "adhocore/jwt", @@ -1187,16 +1187,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.7", + "version": "0.45.8", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "f508c5fcec8e4b2c323a12dd4355a8cb0cc6ad03" + "reference": "8c68f04f073e50c5199bc2df22c5bf9e3be512e4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/f508c5fcec8e4b2c323a12dd4355a8cb0cc6ad03", - "reference": "f508c5fcec8e4b2c323a12dd4355a8cb0cc6ad03", + "url": "https://api.github.com/repos/utopia-php/database/zipball/8c68f04f073e50c5199bc2df22c5bf9e3be512e4", + "reference": "8c68f04f073e50c5199bc2df22c5bf9e3be512e4", "shasum": "" }, "require": { @@ -1237,9 +1237,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.7" + "source": "https://github.com/utopia-php/database/tree/0.45.8" }, - "time": "2024-03-05T10:28:02+00:00" + "time": "2024-03-07T16:36:43+00:00" }, { "name": "utopia-php/domains", From 664df69a5f944b83c99f2367d8386e2aeff8067d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Mar 2024 17:49:59 +0100 Subject: [PATCH 077/406] Fix method refs --- app/cli.php | 8 ++++---- app/controllers/api/projects.php | 4 ++-- app/init.php | 8 ++++---- app/realtime.php | 4 ++-- app/worker.php | 12 ++++++------ .../Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 4 ++-- src/Appwrite/Platform/Workers/Hamster.php | 4 ++-- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/app/cli.php b/app/cli.php index e246101e1c..f03be2cb33 100644 --- a/app/cli.php +++ b/app/cli.php @@ -106,12 +106,12 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } @@ -130,12 +130,12 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 89ca9e15fe..19f8f33a3d 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -201,12 +201,12 @@ App::post('/v1/projects') if ($database === DATABASE_SHARED_TABLES) { $dbForProject - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $dbForProject - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } diff --git a/app/init.php b/app/init.php index a7ffcc5154..cbfe1bfc5c 100644 --- a/app/init.php +++ b/app/init.php @@ -1137,12 +1137,12 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } @@ -1185,12 +1185,12 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } diff --git a/app/realtime.php b/app/realtime.php index 80c5a4fbaf..0e31df50f8 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -80,12 +80,12 @@ function getProjectDB(Document $project): Database if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } diff --git a/app/worker.php b/app/worker.php index 989f1223af..d732cb5ee0 100644 --- a/app/worker.php +++ b/app/worker.php @@ -77,12 +77,12 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } @@ -116,12 +116,12 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } @@ -140,12 +140,12 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso if ($project->getAttribute('database') === DATABASE_SHARED_TABLES) { $database - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $database - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 5fe4b10472..0bb15830fc 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -94,12 +94,12 @@ class DeleteOrphanedProjects extends Action if ($database === DATABASE_SHARED_TABLES) { $dbForProject - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $dbForProject - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index fa47c27c6d..a88b8a4c97 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -106,12 +106,12 @@ class GetMigrationStats extends Action if ($database === DATABASE_SHARED_TABLES) { $dbForProject - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $dbForProject - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php index 167e4d7fe7..aeb243117e 100644 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -126,12 +126,12 @@ class Hamster extends Action if ($database === DATABASE_SHARED_TABLES) { $dbForProject - ->setShareTables(true) + ->setSharedTables(true) ->setTenant($project->getInternalId()) ->setNamespace(''); } else { $dbForProject - ->setShareTables(false) + ->setSharedTables(false) ->setTenant(null) ->setNamespace('_' . $project->getInternalId()); } From 9ca1375af332299f5f283931e3b1f676768e59ec Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 9 Mar 2024 22:56:20 +0100 Subject: [PATCH 078/406] Fix index lengths given tenant is added --- app/config/collections.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index e2f3c11be7..9693b19bf7 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -313,7 +313,7 @@ $commonCollections = [ '$id' => ID::custom('_key_email'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['email'], - 'lengths' => [320], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ @@ -733,7 +733,7 @@ $commonCollections = [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_KEY, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [100, 100], + 'lengths' => [128, 128], 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ @@ -857,14 +857,14 @@ $commonCollections = [ '$id' => ID::custom('_key_userInternalId_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['userInternalId', 'provider', 'providerUid'], - 'lengths' => [Database::LENGTH_KEY, 100, 385], + 'lengths' => [11, 128, 128], 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [100, 640], + 'lengths' => [128, 128], 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ @@ -885,7 +885,7 @@ $commonCollections = [ '$id' => ID::custom('_key_provider'), 'type' => Database::INDEX_KEY, 'attributes' => ['provider'], - 'lengths' => [100], + 'lengths' => [128], 'orders' => [Database::ORDER_ASC], ], [ @@ -2113,7 +2113,7 @@ $projectCollections = array_merge([ '$id' => ID::custom('_key_name'), 'type' => Database::INDEX_KEY, 'attributes' => ['name'], - 'lengths' => [768], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ @@ -2162,7 +2162,7 @@ $projectCollections = array_merge([ '$id' => ID::custom('_key_runtime'), 'type' => Database::INDEX_KEY, 'attributes' => ['runtime'], - 'lengths' => [768], + 'lengths' => [64], 'orders' => [Database::ORDER_ASC], ], [ @@ -2902,14 +2902,14 @@ $projectCollections = array_merge([ '$id' => ID::custom('_key_trigger'), 'type' => Database::INDEX_KEY, 'attributes' => ['trigger'], - 'lengths' => [128], + 'lengths' => [32], 'orders' => [Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_status'), 'type' => Database::INDEX_KEY, 'attributes' => ['status'], - 'lengths' => [128], + 'lengths' => [32], 'orders' => [Database::ORDER_ASC], ], [ @@ -4834,14 +4834,14 @@ $bucketCollections = [ '$id' => ID::custom('_key_name'), 'type' => Database::INDEX_KEY, 'attributes' => ['name'], - 'lengths' => [768], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_signature'), 'type' => Database::INDEX_KEY, 'attributes' => ['signature'], - 'lengths' => [768], + 'lengths' => [256], 'orders' => [Database::ORDER_ASC], ], [ From b409efc492cb68efae0d372f7c375e70e5a7b543 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 10 Mar 2024 08:01:10 +0000 Subject: [PATCH 079/406] fix path --- tests/e2e/Services/Databases/DatabasesBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 79a6c7a329..c09f9a1cc1 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -4266,7 +4266,7 @@ trait DatabasesBase ], $this->getHeaders()), [ 'documentId' => ID::unique(), 'data' => [ - 'longtext' => file_get_contents(__DIR__ . '/../../../extensionsresources/longtext.txt'), + 'longtext' => file_get_contents(__DIR__ . '/../../../resources/longtext.txt'), ], 'permissions' => [ Permission::read(Role::user($this->getUser()['$id'])), From 65c21a34c0383359856dacc0710983eb2966de26 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 10 Mar 2024 12:50:51 +0100 Subject: [PATCH 080/406] Update database --- composer.json | 2 +- composer.lock | 52 +++++++++++++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/composer.json b/composer.json index e5aef1ef6d..afa8520cfb 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "utopia-php/cache": "0.9.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.45.8", + "utopia-php/database": "0.45.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index b0046320f9..8fa95ff28f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "767fe773a75b04ff6c22eac8b2c1498b", + "content-hash": "d9a7df9d8d4ed11e157accbd2d73bf7b", "packages": [ { "name": "adhocore/jwt", @@ -404,16 +404,16 @@ }, { "name": "jean85/pretty-package-versions", - "version": "2.0.5", + "version": "2.0.6", "source": { "type": "git", "url": "https://github.com/Jean85/pretty-package-versions.git", - "reference": "ae547e455a3d8babd07b96966b17d7fd21d9c6af" + "reference": "f9fdd29ad8e6d024f52678b570e5593759b550b4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/ae547e455a3d8babd07b96966b17d7fd21d9c6af", - "reference": "ae547e455a3d8babd07b96966b17d7fd21d9c6af", + "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/f9fdd29ad8e6d024f52678b570e5593759b550b4", + "reference": "f9fdd29ad8e6d024f52678b570e5593759b550b4", "shasum": "" }, "require": { @@ -421,9 +421,9 @@ "php": "^7.1|^8.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^2.17", + "friendsofphp/php-cs-fixer": "^3.2", "jean85/composer-provided-replaced-stub-package": "^1.0", - "phpstan/phpstan": "^0.12.66", + "phpstan/phpstan": "^1.4", "phpunit/phpunit": "^7.5|^8.5|^9.4", "vimeo/psalm": "^4.3" }, @@ -457,9 +457,9 @@ ], "support": { "issues": "https://github.com/Jean85/pretty-package-versions/issues", - "source": "https://github.com/Jean85/pretty-package-versions/tree/2.0.5" + "source": "https://github.com/Jean85/pretty-package-versions/tree/2.0.6" }, - "time": "2021-10-08T21:21:46+00:00" + "time": "2024-03-08T09:58:59+00:00" }, { "name": "league/csv", @@ -1187,16 +1187,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.8", + "version": "0.45.9", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "8c68f04f073e50c5199bc2df22c5bf9e3be512e4" + "reference": "f28a3f5b5276d3459694669f01a382b3e35b2006" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/8c68f04f073e50c5199bc2df22c5bf9e3be512e4", - "reference": "8c68f04f073e50c5199bc2df22c5bf9e3be512e4", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f28a3f5b5276d3459694669f01a382b3e35b2006", + "reference": "f28a3f5b5276d3459694669f01a382b3e35b2006", "shasum": "" }, "require": { @@ -1237,9 +1237,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.8" + "source": "https://github.com/utopia-php/database/tree/0.45.9" }, - "time": "2024-03-07T16:36:43+00:00" + "time": "2024-03-10T11:46:14+00:00" }, { "name": "utopia-php/domains", @@ -2413,30 +2413,30 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.36.4", + "version": "0.36.7", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "8d932098009d62d37dda73cfe4ebc11f83e21405" + "reference": "c9c07e8d96b80f62507bbb5ef90fa0769e560621" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/8d932098009d62d37dda73cfe4ebc11f83e21405", - "reference": "8d932098009d62d37dda73cfe4ebc11f83e21405", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/c9c07e8d96b80f62507bbb5ef90fa0769e560621", + "reference": "c9c07e8d96b80f62507bbb5ef90fa0769e560621", "shasum": "" }, "require": { "ext-curl": "*", "ext-json": "*", "ext-mbstring": "*", - "matthiasmullie/minify": "^1.3.68", + "matthiasmullie/minify": "1.3.*", "php": ">=8.0", - "twig/twig": "^3.4.1" + "twig/twig": "v3.8.*" }, "require-dev": { - "brianium/paratest": "^6.4", - "phpunit/phpunit": "^9.5.21", - "squizlabs/php_codesniffer": "^3.6" + "brianium/paratest": "v7.4.*", + "phpunit/phpunit": "10.5.*", + "squizlabs/php_codesniffer": "3.9.*" }, "type": "library", "autoload": { @@ -2458,9 +2458,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.36.4" + "source": "https://github.com/appwrite/sdk-generator/tree/0.36.7" }, - "time": "2024-02-20T16:36:15+00:00" + "time": "2024-03-09T16:57:18+00:00" }, { "name": "doctrine/deprecations", From 7fc990a56d7cc2f86c8854e9279e1635b72764bf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 10 Mar 2024 18:45:50 +0100 Subject: [PATCH 081/406] Debug --- app/controllers/shared/api.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a5f5af764e..8454e41b50 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -523,7 +523,9 @@ App::shutdown() $queueForDeletes->trigger(); } + \var_dump('About to trigger database event'); if (!empty($queueForDatabase->getType())) { + \var_dump('Triggering database event'); $queueForDatabase->trigger(); } From 15a6c510a01ba92d1edf89b3ab2eac3ff6f13514 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 11 Mar 2024 10:23:52 +0100 Subject: [PATCH 082/406] Upload 400s to separate error logger --- app/controllers/general.php | 18 ++++++++++++++++++ docker-compose.yml | 2 ++ 2 files changed, 20 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index e945189b24..073001e54b 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -793,6 +793,24 @@ App::error() $publish = $error->getCode() === 0 || $error->getCode() >= 500; } + //TEMP, After 13/03/2024 remove. + if ($error->getCode() >= 400 && $error->getCode() < 500) { + // Register error logger + $providerName = App::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', ''); + $providerConfig = App::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', ''); + + if (!(empty($providerName) || empty($providerConfig))) { + if (!Logger::hasProvider($providerName)) { + throw new Exception("Logging provider not supported. Logging is disabled"); + } + + $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName); + $adapter = new $classname($providerConfig); + $logger = new Logger($adapter); + $publish = true; + } + } + if ($logger && ($publish || $error->getCode() === 0)) { try { /** @var Utopia\Database\Document $user */ diff --git a/docker-compose.yml b/docker-compose.yml index 3d4c5d57be..26c5301526 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -185,6 +185,8 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY + - _APP_EXPERIMENT_LOGGING_PROVIDER + - _APP_EXPERIMENT_LOGGING_CONFIG appwrite-realtime: entrypoint: realtime From 9c100758deb3f342ef9612099fe266278190818e Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 11 Mar 2024 10:29:57 +0100 Subject: [PATCH 083/406] Update general.php --- app/controllers/general.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 073001e54b..86ad0a8d8e 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -793,7 +793,6 @@ App::error() $publish = $error->getCode() === 0 || $error->getCode() >= 500; } - //TEMP, After 13/03/2024 remove. if ($error->getCode() >= 400 && $error->getCode() < 500) { // Register error logger $providerName = App::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', ''); From cea4e0d8b61953a719d3612df1803d1380d41f64 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 11 Mar 2024 12:54:42 +0100 Subject: [PATCH 084/406] Wrap all db exceptions in error handler --- app/controllers/api/databases.php | 79 ++++--------- app/controllers/api/storage.php | 190 +++++++++++++----------------- app/controllers/api/teams.php | 9 +- app/controllers/general.php | 18 ++- 4 files changed, 124 insertions(+), 172 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 7542b24d35..6789fa57c0 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -662,8 +662,6 @@ App::put('/v1/databases/:databaseId') ->setAttribute('name', $name) ->setAttribute('enabled', $enabled) ->setAttribute('search', implode(' ', [$databaseId, $name]))); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); } catch (StructureException $exception) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage()); } @@ -1015,19 +1013,14 @@ App::put('/v1/databases/:databaseId/collections/:collectionId') $enabled ??= $collection->getAttribute('enabled', true); - try { - $collection = $dbForProject->updateDocument('database_' . $database->getInternalId(), $collectionId, $collection - ->setAttribute('name', $name) - ->setAttribute('$permissions', $permissions) - ->setAttribute('documentSecurity', $documentSecurity) - ->setAttribute('enabled', $enabled) - ->setAttribute('search', implode(' ', [$collectionId, $name]))); - $dbForProject->updateCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $permissions, $documentSecurity); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (StructureException $exception) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage()); - } + $collection = $dbForProject->updateDocument('database_' . $database->getInternalId(), $collectionId, $collection + ->setAttribute('name', $name) + ->setAttribute('$permissions', $permissions) + ->setAttribute('documentSecurity', $documentSecurity) + ->setAttribute('enabled', $enabled) + ->setAttribute('search', implode(' ', [$collectionId, $name]))); + + $dbForProject->updateCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $permissions, $documentSecurity); $queueForEvents ->setContext('database', $database) @@ -2936,14 +2929,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $filters = Query::groupByType($queries)['filters']; - try { - $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); - $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $filters, APP_LIMIT_COUNT); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, $e->getMessage()); - } + $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); + $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $filters, APP_LIMIT_COUNT); // Add $collectionId and $databaseId for all documents $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { @@ -3069,13 +3056,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queries = Query::parseQueries($queries); - try { - $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, $e->getMessage()); - } + $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); @@ -3387,22 +3368,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); - try { - $document = $dbForProject->withRequestTimestamp( - $requestTimestamp, - fn() => $dbForProject->updateDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $document->getId(), - $newDocument - ) - ); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (DuplicateException) { - throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS); - } catch (StructureException $exception) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage()); - } + $document = $dbForProject->withRequestTimestamp( + $requestTimestamp, + fn() => $dbForProject->updateDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $document->getId(), + $newDocument + ) + ); // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { @@ -3502,16 +3475,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) { - try { - $dbForProject->deleteDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $documentId - ); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (RestrictedException) { - throw new Exception(Exception::DOCUMENT_DELETE_RESTRICTED); - } + $dbForProject->deleteDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $documentId + ); }); // Add $collectionId and $databaseId for all documents diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index efe2515468..26ae0adacf 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -581,111 +581,95 @@ App::post('/v1/storage/buckets/:bucketId/files') $openSSLIV = \bin2hex($iv); } - try { - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => $fileId, - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getInternalId(), - 'name' => $fileName, - 'path' => $path, - 'signature' => $fileHash, - 'mimeType' => $mimeType, - 'sizeOriginal' => $fileSize, - 'sizeActual' => $sizeActual, - 'algorithm' => $algorithm, - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'openSSLVersion' => $openSSLVersion, - 'openSSLCipher' => $openSSLCipher, - 'openSSLTag' => $openSSLTag, - 'openSSLIV' => $openSSLIV, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getInternalId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $fileSize, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); - $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); - } else { - $file = $file - ->setAttribute('$permissions', $permissions) - ->setAttribute('signature', $fileHash) - ->setAttribute('mimeType', $mimeType) - ->setAttribute('sizeActual', $sizeActual) - ->setAttribute('algorithm', $algorithm) - ->setAttribute('openSSLVersion', $openSSLVersion) - ->setAttribute('openSSLCipher', $openSSLCipher) - ->setAttribute('openSSLTag', $openSSLTag) - ->setAttribute('openSSLIV', $openSSLIV) - ->setAttribute('metadata', $metadata) - ->setAttribute('chunksUploaded', $chunksUploaded); + $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); + } else { + $file = $file + ->setAttribute('$permissions', $permissions) + ->setAttribute('signature', $fileHash) + ->setAttribute('mimeType', $mimeType) + ->setAttribute('sizeActual', $sizeActual) + ->setAttribute('algorithm', $algorithm) + ->setAttribute('openSSLVersion', $openSSLVersion) + ->setAttribute('openSSLCipher', $openSSLCipher) + ->setAttribute('openSSLTag', $openSSLTag) + ->setAttribute('openSSLIV', $openSSLIV) + ->setAttribute('metadata', $metadata) + ->setAttribute('chunksUploaded', $chunksUploaded); - /** - * Validate create permission and skip authorization in updateDocument - * Without this, the file creation will fail when user doesn't have update permission - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we validate create permission instead of update - */ - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); } - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (StructureException $exception) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage()); - } catch (DuplicateException) { - throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS); + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); } } else { - try { - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => ID::custom($fileId), - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getInternalId(), - 'name' => $fileName, - 'path' => $path, - 'signature' => '', - 'mimeType' => '', - 'sizeOriginal' => $fileSize, - 'sizeActual' => 0, - 'algorithm' => '', - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getInternalId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); - $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); - } else { - $file = $file - ->setAttribute('chunksUploaded', $chunksUploaded) - ->setAttribute('metadata', $metadata); + $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); + } else { + $file = $file + ->setAttribute('chunksUploaded', $chunksUploaded) + ->setAttribute('metadata', $metadata); - /** - * Validate create permission and skip authorization in updateDocument - * Without this, the file creation will fail when user doesn't have update permission - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we validate create permission instead of update - */ - $validator = new Authorization(Database::PERMISSION_CREATE); - if (!$validator->isValid($bucket->getCreate())) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } - $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); + /** + * Validate create permission and skip authorization in updateDocument + * Without this, the file creation will fail when user doesn't have update permission + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we validate create permission instead of update + */ + $validator = new Authorization(Database::PERMISSION_CREATE); + if (!$validator->isValid($bucket->getCreate())) { + throw new Exception(Exception::USER_UNAUTHORIZED); } - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (StructureException $exception) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage()); - } catch (DuplicateException) { - throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS); + $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); } } @@ -1383,11 +1367,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') } if ($fileSecurity && !$valid) { - try { - $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } + $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); } else { $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); } @@ -1471,11 +1451,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ; if ($fileSecurity && !$valid) { - try { - $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId); - } catch (AuthorizationException) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } + $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId); } else { $deleted = Authorization::skip(fn() => $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId)); } diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 2801e45bb5..308f19e45e 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1036,14 +1036,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') throw new Exception(Exception::TEAM_NOT_FOUND); } - try { - $dbForProject->deleteDocument('memberships', $membership->getId()); - } catch (AuthorizationException $exception) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } catch (\Exception $exception) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove membership from DB'); - } - + $dbForProject->deleteDocument('memberships', $membership->getId()); $dbForProject->deleteCachedDocument('users', $user->getId()); if ($membership->getAttribute('confirm')) { // Count only confirmed members diff --git a/app/controllers/general.php b/app/controllers/general.php index 86ad0a8d8e..565d6ff42b 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -810,7 +810,7 @@ App::error() } } - if ($logger && ($publish || $error->getCode() === 0)) { + if ($logger && $publish) { try { /** @var Utopia\Database\Document $user */ $user = $utopia->getResource('user'); @@ -892,6 +892,22 @@ App::error() $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error); $code = $error->getCode(); $message = $error->getMessage(); + } elseif ($error instanceof Utopia\Database\Exception\Query) { + $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, previous: $error); + $code = $error->getCode(); + $message = $error->getMessage(); + } elseif ($error instanceof Utopia\Database\Exception\Structure) { + $error = new AppwriteException(AppwriteException::DOCUMENT_INVALID_STRUCTURE, $error->getMessage(), previous: $error); + $code = $error->getCode(); + $message = $error->getMessage(); + } elseif ($error instanceof Utopia\Database\Exception\Duplicate) { + $error = new AppwriteException(AppwriteException::DOCUMENT_ALREADY_EXISTS); + $code = $error->getCode(); + $message = $error->getMessage(); + } elseif ($error instanceof Utopia\Database\Exception\Restricted) { + $error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED); + $code = $error->getCode(); + $message = $error->getMessage(); } /** Wrap all exceptions inside Appwrite\Extend\Exception */ From 22b3ea7e8dda3711883b653c8c23fe5c7b15db9e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 11 Mar 2024 13:03:30 +0100 Subject: [PATCH 085/406] Catch auth fixes --- app/controllers/api/databases.php | 21 ++++----------------- app/controllers/general.php | 4 ++++ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 6789fa57c0..e58e1f211f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -19,16 +19,13 @@ use Utopia\App; use Utopia\Audit\Audit; use Utopia\Config\Config; use Utopia\Database\Database; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; -use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Restricted as RestrictedException; use Utopia\Database\Exception\Structure as StructureException; -use Utopia\Database\Exception\Timeout as TimeoutException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; @@ -657,14 +654,10 @@ App::put('/v1/databases/:databaseId') throw new Exception(Exception::DATABASE_NOT_FOUND); } - try { - $database = $dbForProject->updateDocument('databases', $databaseId, $database - ->setAttribute('name', $name) - ->setAttribute('enabled', $enabled) - ->setAttribute('search', implode(' ', [$databaseId, $name]))); - } catch (StructureException $exception) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage()); - } + $database = $dbForProject->updateDocument('databases', $databaseId, $database + ->setAttribute('name', $name) + ->setAttribute('enabled', $enabled) + ->setAttribute('search', implode(' ', [$databaseId, $name]))); $queueForEvents->setParam('databaseId', $database->getId()); @@ -2815,13 +2808,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); - try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); - } catch (StructureException $exception) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage()); - } catch (DuplicateException $exception) { - throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS); - } // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { diff --git a/app/controllers/general.php b/app/controllers/general.php index 565d6ff42b..490a7270f5 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -908,6 +908,10 @@ App::error() $error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED); $code = $error->getCode(); $message = $error->getMessage(); + } elseif ($error instanceof Utopia\Database\Exception\Authorization) { + $error = new AppwriteException(AppwriteException::USER_UNAUTHORIZED); + $code = $error->getCode(); + $message = $error->getMessage(); } /** Wrap all exceptions inside Appwrite\Extend\Exception */ From 550837b572956ec62a30b849de5af36a4b4b8581 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 11 Mar 2024 13:46:02 +0100 Subject: [PATCH 086/406] Pass through query exception message --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 490a7270f5..afb8c84684 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -893,7 +893,7 @@ App::error() $code = $error->getCode(); $message = $error->getMessage(); } elseif ($error instanceof Utopia\Database\Exception\Query) { - $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, previous: $error); + $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, $error->getMessage(), previous: $error); $code = $error->getCode(); $message = $error->getMessage(); } elseif ($error instanceof Utopia\Database\Exception\Structure) { From 7e1894feb57a561cfeee263aaecc315734614b05 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 11 Mar 2024 14:14:32 +0100 Subject: [PATCH 087/406] Debug --- app/controllers/shared/api.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8454e41b50..9e69ae4ced 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -523,10 +523,10 @@ App::shutdown() $queueForDeletes->trigger(); } - \var_dump('About to trigger database event'); if (!empty($queueForDatabase->getType())) { - \var_dump('Triggering database event'); + \var_dump('Before triggering database event'); $queueForDatabase->trigger(); + \var_dump('After triggering database event'); } /** From a1d86c9b3f1df481a11e077fe84c4103ec5fb6bb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 11 Mar 2024 16:11:37 +0100 Subject: [PATCH 088/406] Switch over errors --- app/controllers/general.php | 76 ++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index afb8c84684..85f80aa2a4 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -853,6 +853,7 @@ App::error() Console::info('Log pushed with status code: ' . $responseCode); } + $class = \get_class($error); $code = $error->getCode(); $message = $error->getMessage(); $file = $error->getFile(); @@ -873,47 +874,44 @@ App::error() Console::error('[Error] Line: ' . $line); } - /** Handle Utopia Errors */ - if ($error instanceof Utopia\Exception) { - $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error); - switch ($code) { - case 400: - $error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID); - break; - case 404: - $error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND); - break; - } - } elseif ($error instanceof Utopia\Database\Exception\Conflict) { - $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error); - $code = $error->getCode(); - $message = $error->getMessage(); - } elseif ($error instanceof Utopia\Database\Exception\Timeout) { - $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error); - $code = $error->getCode(); - $message = $error->getMessage(); - } elseif ($error instanceof Utopia\Database\Exception\Query) { - $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, $error->getMessage(), previous: $error); - $code = $error->getCode(); - $message = $error->getMessage(); - } elseif ($error instanceof Utopia\Database\Exception\Structure) { - $error = new AppwriteException(AppwriteException::DOCUMENT_INVALID_STRUCTURE, $error->getMessage(), previous: $error); - $code = $error->getCode(); - $message = $error->getMessage(); - } elseif ($error instanceof Utopia\Database\Exception\Duplicate) { - $error = new AppwriteException(AppwriteException::DOCUMENT_ALREADY_EXISTS); - $code = $error->getCode(); - $message = $error->getMessage(); - } elseif ($error instanceof Utopia\Database\Exception\Restricted) { - $error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED); - $code = $error->getCode(); - $message = $error->getMessage(); - } elseif ($error instanceof Utopia\Database\Exception\Authorization) { - $error = new AppwriteException(AppwriteException::USER_UNAUTHORIZED); - $code = $error->getCode(); - $message = $error->getMessage(); + switch ($class) { + case 'Utopia\Exception': + $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error); + switch ($code) { + case 400: + $error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID); + break; + case 404: + $error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND); + break; + } + break; + case 'Utopia\Database\Exception\Conflict': + $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error); + break; + case 'Utopia\Database\Exception\Timeout': + $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error); + break; + case 'Utopia\Database\Exception\Query': + $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, $error->getMessage(), previous: $error); + break; + case 'Utopia\Database\Exception\Structure': + $error = new AppwriteException(AppwriteException::DOCUMENT_INVALID_STRUCTURE, $error->getMessage(), previous: $error); + break; + case 'Utopia\Database\Exception\Duplicate': + $error = new AppwriteException(AppwriteException::DOCUMENT_ALREADY_EXISTS); + break; + case 'Utopia\Database\Exception\Restricted': + $error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED); + break; + case 'Utopia\Database\Exception\Authorization': + $error = new AppwriteException(AppwriteException::USER_UNAUTHORIZED); + break; } + $code = $error->getCode(); + $message = $error->getMessage(); + /** Wrap all exceptions inside Appwrite\Extend\Exception */ if (!($error instanceof AppwriteException)) { $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error); From cfc69b0f9212eb950afc7849b502bfed3775083a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 12 Mar 2024 16:54:06 +0100 Subject: [PATCH 089/406] Debug --- src/Appwrite/Event/Database.php | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index 442cbe4bbc..c2a36577c4 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -111,14 +111,23 @@ class Database extends Event $client = new Client($this->queue, $this->connection); - return $client->enqueue([ - 'project' => $this->project, - 'user' => $this->user, - 'type' => $this->type, - 'collection' => $this->collection, - 'document' => $this->document, - 'database' => $this->database, - 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); + try { + $result = $client->enqueue([ + 'project' => $this->project, + 'user' => $this->user, + 'type' => $this->type, + 'collection' => $this->collection, + 'document' => $this->document, + 'database' => $this->database, + 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) + ]); + \var_dump('Enqueued event'); + \var_dump($result); + return $result; + } catch (\Throwable $th) { + \var_dump('Enqueue event failed'); + \var_dump($th); + return false; + } } } From c7bf9bb3ec3d8c28bfaa978000894afd78b1df01 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 12 Mar 2024 17:09:09 +0100 Subject: [PATCH 090/406] Debug --- src/Appwrite/Event/Database.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index c2a36577c4..ace846d8b4 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -111,6 +111,8 @@ class Database extends Event $client = new Client($this->queue, $this->connection); + \var_dump('Event queue name is: ' . $this->queue); + try { $result = $client->enqueue([ 'project' => $this->project, From b4c065a342804c8a602018f0c35cbb3938539d08 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 13 Mar 2024 01:03:51 +0100 Subject: [PATCH 091/406] Escape function build command The extra trim is neded to because the extra quotes interfere with additional operations in the executor/orchestration library. --- src/Appwrite/Platform/Workers/Builds.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 120a514fc4..1bc297d55b 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -385,7 +385,6 @@ class Builds extends Action ]); $command = $deployment->getAttribute('commands', ''); - $command = \str_replace('"', '\\"', $command); $response = null; $err = null; @@ -394,7 +393,7 @@ class Builds extends Action Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) { try { $version = $function->getAttribute('version', 'v2'); - $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"'; + $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"'; $response = $executor->createRuntime( deploymentId: $deployment->getId(), From a4bdf63afd4f9c099026513519c329092a103c4b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 13 Mar 2024 12:41:55 +0100 Subject: [PATCH 092/406] Fix exception wrap order of operations --- app/controllers/general.php | 126 +++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 85f80aa2a4..11fabe7550 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -786,6 +786,71 @@ App::error() ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); + $class = \get_class($error); + $code = $error->getCode(); + $message = $error->getMessage(); + $file = $error->getFile(); + $line = $error->getLine(); + $trace = $error->getTrace(); + + if (php_sapi_name() === 'cli') { + Console::error('[Error] Timestamp: ' . date('c', time())); + + if ($route) { + Console::error('[Error] Method: ' . $route->getMethod()); + Console::error('[Error] URL: ' . $route->getPath()); + } + + Console::error('[Error] Type: ' . get_class($error)); + Console::error('[Error] Message: ' . $message); + Console::error('[Error] File: ' . $file); + Console::error('[Error] Line: ' . $line); + } + + switch ($class) { + case 'Utopia\Exception': + $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error); + switch ($code) { + case 400: + $error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID); + break; + case 404: + $error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND); + break; + } + break; + case 'Utopia\Database\Exception\Conflict': + \var_dump('Wrapping conflict exception'); + $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error); + break; + case 'Utopia\Database\Exception\Timeout': + \var_dump('Wrapping timeout exception'); + $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error); + break; + case 'Utopia\Database\Exception\Query': + \var_dump('Wrapping query exception'); + $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, $error->getMessage(), previous: $error); + break; + case 'Utopia\Database\Exception\Structure': + \var_dump('Wrapping structure exception'); + $error = new AppwriteException(AppwriteException::DOCUMENT_INVALID_STRUCTURE, $error->getMessage(), previous: $error); + break; + case 'Utopia\Database\Exception\Duplicate': + \var_dump('Wrapping duplicate exception'); + $error = new AppwriteException(AppwriteException::DOCUMENT_ALREADY_EXISTS); + break; + case 'Utopia\Database\Exception\Restricted': + \var_dump('Wrapping restricted exception'); + $error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED); + break; + case 'Utopia\Database\Exception\Authorization': + \var_dump('Wrapping authorization exception'); + $error = new AppwriteException(AppwriteException::USER_UNAUTHORIZED); + break; + } + + $code = $error->getCode(); + $message = $error->getMessage(); if ($error instanceof AppwriteException) { $publish = $error->isPublishable(); @@ -814,7 +879,7 @@ App::error() try { /** @var Utopia\Database\Document $user */ $user = $utopia->getResource('user'); - } catch (\Throwable $th) { + } catch (\Throwable) { // All good, user is optional information for logger } @@ -853,65 +918,6 @@ App::error() Console::info('Log pushed with status code: ' . $responseCode); } - $class = \get_class($error); - $code = $error->getCode(); - $message = $error->getMessage(); - $file = $error->getFile(); - $line = $error->getLine(); - $trace = $error->getTrace(); - - if (php_sapi_name() === 'cli') { - Console::error('[Error] Timestamp: ' . date('c', time())); - - if ($route) { - Console::error('[Error] Method: ' . $route->getMethod()); - Console::error('[Error] URL: ' . $route->getPath()); - } - - Console::error('[Error] Type: ' . get_class($error)); - Console::error('[Error] Message: ' . $message); - Console::error('[Error] File: ' . $file); - Console::error('[Error] Line: ' . $line); - } - - switch ($class) { - case 'Utopia\Exception': - $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error); - switch ($code) { - case 400: - $error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID); - break; - case 404: - $error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND); - break; - } - break; - case 'Utopia\Database\Exception\Conflict': - $error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error); - break; - case 'Utopia\Database\Exception\Timeout': - $error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error); - break; - case 'Utopia\Database\Exception\Query': - $error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, $error->getMessage(), previous: $error); - break; - case 'Utopia\Database\Exception\Structure': - $error = new AppwriteException(AppwriteException::DOCUMENT_INVALID_STRUCTURE, $error->getMessage(), previous: $error); - break; - case 'Utopia\Database\Exception\Duplicate': - $error = new AppwriteException(AppwriteException::DOCUMENT_ALREADY_EXISTS); - break; - case 'Utopia\Database\Exception\Restricted': - $error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED); - break; - case 'Utopia\Database\Exception\Authorization': - $error = new AppwriteException(AppwriteException::USER_UNAUTHORIZED); - break; - } - - $code = $error->getCode(); - $message = $error->getMessage(); - /** Wrap all exceptions inside Appwrite\Extend\Exception */ if (!($error instanceof AppwriteException)) { $error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error); From 310aa56e5df0dafa429effe7ee2410e9c7bc97ac Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 13 Mar 2024 19:08:07 +0100 Subject: [PATCH 093/406] Update db --- composer.json | 2 +- composer.lock | 164 +++++++++++++++++++++++++++----------------------- 2 files changed, 89 insertions(+), 77 deletions(-) diff --git a/composer.json b/composer.json index 4e98d868d4..faf30a117d 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "utopia-php/cache": "0.9.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.45.*", + "utopia-php/database": "0.45.7", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index c8fe99a704..c7fd4959b5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f7a4173bb5adeea3518f949fa883a282", + "content-hash": "9492cb777590ff776c2a73620783d74b", "packages": [ { "name": "adhocore/jwt", @@ -404,16 +404,16 @@ }, { "name": "jean85/pretty-package-versions", - "version": "2.0.5", + "version": "2.0.6", "source": { "type": "git", "url": "https://github.com/Jean85/pretty-package-versions.git", - "reference": "ae547e455a3d8babd07b96966b17d7fd21d9c6af" + "reference": "f9fdd29ad8e6d024f52678b570e5593759b550b4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/ae547e455a3d8babd07b96966b17d7fd21d9c6af", - "reference": "ae547e455a3d8babd07b96966b17d7fd21d9c6af", + "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/f9fdd29ad8e6d024f52678b570e5593759b550b4", + "reference": "f9fdd29ad8e6d024f52678b570e5593759b550b4", "shasum": "" }, "require": { @@ -421,9 +421,9 @@ "php": "^7.1|^8.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^2.17", + "friendsofphp/php-cs-fixer": "^3.2", "jean85/composer-provided-replaced-stub-package": "^1.0", - "phpstan/phpstan": "^0.12.66", + "phpstan/phpstan": "^1.4", "phpunit/phpunit": "^7.5|^8.5|^9.4", "vimeo/psalm": "^4.3" }, @@ -457,9 +457,9 @@ ], "support": { "issues": "https://github.com/Jean85/pretty-package-versions/issues", - "source": "https://github.com/Jean85/pretty-package-versions/tree/2.0.5" + "source": "https://github.com/Jean85/pretty-package-versions/tree/2.0.6" }, - "time": "2021-10-08T21:21:46+00:00" + "time": "2024-03-08T09:58:59+00:00" }, { "name": "league/csv", @@ -688,7 +688,7 @@ "version": "0.6.3", "source": { "type": "git", - "url": "git@github.com:mustangostang/spyc.git", + "url": "https://github.com/mustangostang/spyc.git", "reference": "4627c838b16550b666d15aeae1e5289dd5b77da0" }, "dist": { @@ -731,6 +731,10 @@ "yaml", "yml" ], + "support": { + "issues": "https://github.com/mustangostang/spyc/issues", + "source": "https://github.com/mustangostang/spyc/tree/0.6.3" + }, "time": "2019-09-10T13:16:29+00:00" }, { @@ -1187,16 +1191,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.6", + "version": "0.45.7", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd" + "reference": "3ffcc226cbc5b8747cdc5d6b06ce887cdffee8e1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd", - "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd", + "url": "https://api.github.com/repos/utopia-php/database/zipball/3ffcc226cbc5b8747cdc5d6b06ce887cdffee8e1", + "reference": "3ffcc226cbc5b8747cdc5d6b06ce887cdffee8e1", "shasum": "" }, "require": { @@ -1237,9 +1241,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.6" + "source": "https://github.com/utopia-php/database/tree/0.45.7" }, - "time": "2024-02-01T02:33:43+00:00" + "time": "2024-03-13T18:03:15+00:00" }, { "name": "utopia-php/domains", @@ -1350,16 +1354,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.2", + "version": "0.33.3", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3" + "reference": "893d602cd96676810c25fc9b9a2e9360eebb898f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", - "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", + "url": "https://api.github.com/repos/utopia-php/http/zipball/893d602cd96676810c25fc9b9a2e9360eebb898f", + "reference": "893d602cd96676810c25fc9b9a2e9360eebb898f", "shasum": "" }, "require": { @@ -1389,9 +1393,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.2" + "source": "https://github.com/utopia-php/http/tree/0.33.3" }, - "time": "2024-01-31T10:35:59+00:00" + "time": "2024-03-11T13:43:23+00:00" }, { "name": "utopia-php/image", @@ -2581,16 +2585,16 @@ }, { "name": "matthiasmullie/minify", - "version": "1.3.71", + "version": "1.3.72", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "ae42a47d7fecc1fbb7277b2f2d84c37a33edc3b1" + "reference": "531fdeef1911ffe27a53f8a19c297648c78f757e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/ae42a47d7fecc1fbb7277b2f2d84c37a33edc3b1", - "reference": "ae42a47d7fecc1fbb7277b2f2d84c37a33edc3b1", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/531fdeef1911ffe27a53f8a19c297648c78f757e", + "reference": "531fdeef1911ffe27a53f8a19c297648c78f757e", "shasum": "" }, "require": { @@ -2640,7 +2644,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.71" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.72" }, "funding": [ { @@ -2648,7 +2652,7 @@ "type": "github" } ], - "time": "2023-04-25T20:33:03+00:00" + "time": "2024-03-13T12:02:00+00:00" }, { "name": "matthiasmullie/path-converter", @@ -2764,16 +2768,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.0.1", + "version": "v5.0.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "2218c2252c874a4624ab2f613d86ac32d227bc69" + "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/2218c2252c874a4624ab2f613d86ac32d227bc69", - "reference": "2218c2252c874a4624ab2f613d86ac32d227bc69", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/139676794dc1e9231bf7bcd123cfc0c99182cb13", + "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13", "shasum": "" }, "require": { @@ -2816,26 +2820,27 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.2" }, - "time": "2024-02-21T19:24:10+00:00" + "time": "2024-03-05T20:51:40+00:00" }, { "name": "phar-io/manifest", - "version": "2.0.3", + "version": "2.0.4", "source": { "type": "git", "url": "https://github.com/phar-io/manifest.git", - "reference": "97803eca37d319dfa7826cc2437fc020857acb53" + "reference": "54750ef60c58e43759730615a392c31c80e23176" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/manifest/zipball/97803eca37d319dfa7826cc2437fc020857acb53", - "reference": "97803eca37d319dfa7826cc2437fc020857acb53", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/54750ef60c58e43759730615a392c31c80e23176", + "reference": "54750ef60c58e43759730615a392c31c80e23176", "shasum": "" }, "require": { "ext-dom": "*", + "ext-libxml": "*", "ext-phar": "*", "ext-xmlwriter": "*", "phar-io/version": "^3.0.1", @@ -2876,9 +2881,15 @@ "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", "support": { "issues": "https://github.com/phar-io/manifest/issues", - "source": "https://github.com/phar-io/manifest/tree/2.0.3" + "source": "https://github.com/phar-io/manifest/tree/2.0.4" }, - "time": "2021-07-20T11:28:43+00:00" + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2024-03-03T12:33:53+00:00" }, { "name": "phar-io/version", @@ -3217,16 +3228,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.30", + "version": "9.2.31", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089" + "reference": "48c34b5d8d983006bd2adc2d0de92963b9155965" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/ca2bd87d2f9215904682a9cb9bb37dda98e76089", - "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/48c34b5d8d983006bd2adc2d0de92963b9155965", + "reference": "48c34b5d8d983006bd2adc2d0de92963b9155965", "shasum": "" }, "require": { @@ -3283,7 +3294,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.30" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.31" }, "funding": [ { @@ -3291,7 +3302,7 @@ "type": "github" } ], - "time": "2023-12-22T06:47:57+00:00" + "time": "2024-03-02T06:37:42+00:00" }, { "name": "phpunit/php-file-iterator", @@ -3689,16 +3700,16 @@ }, { "name": "sebastian/cli-parser", - "version": "1.0.1", + "version": "1.0.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/cli-parser.git", - "reference": "442e7c7e687e42adc03470c7b668bc4b2402c0b2" + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/442e7c7e687e42adc03470c7b668bc4b2402c0b2", - "reference": "442e7c7e687e42adc03470c7b668bc4b2402c0b2", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/2b56bea83a09de3ac06bb18b92f068e60cc6f50b", + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b", "shasum": "" }, "require": { @@ -3733,7 +3744,7 @@ "homepage": "https://github.com/sebastianbergmann/cli-parser", "support": { "issues": "https://github.com/sebastianbergmann/cli-parser/issues", - "source": "https://github.com/sebastianbergmann/cli-parser/tree/1.0.1" + "source": "https://github.com/sebastianbergmann/cli-parser/tree/1.0.2" }, "funding": [ { @@ -3741,7 +3752,7 @@ "type": "github" } ], - "time": "2020-09-28T06:08:49+00:00" + "time": "2024-03-02T06:27:43+00:00" }, { "name": "sebastian/code-unit", @@ -3987,16 +3998,16 @@ }, { "name": "sebastian/diff", - "version": "4.0.5", + "version": "4.0.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131" + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/74be17022044ebaaecfdf0c5cd504fc9cd5a7131", - "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/ba01945089c3a293b01ba9badc29ad55b106b0bc", + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc", "shasum": "" }, "require": { @@ -4041,7 +4052,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/diff/issues", - "source": "https://github.com/sebastianbergmann/diff/tree/4.0.5" + "source": "https://github.com/sebastianbergmann/diff/tree/4.0.6" }, "funding": [ { @@ -4049,7 +4060,7 @@ "type": "github" } ], - "time": "2023-05-07T05:35:17+00:00" + "time": "2024-03-02T06:30:58+00:00" }, { "name": "sebastian/environment", @@ -4116,16 +4127,16 @@ }, { "name": "sebastian/exporter", - "version": "4.0.5", + "version": "4.0.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "ac230ed27f0f98f597c8a2b6eb7ac563af5e5b9d" + "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/ac230ed27f0f98f597c8a2b6eb7ac563af5e5b9d", - "reference": "ac230ed27f0f98f597c8a2b6eb7ac563af5e5b9d", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/78c00df8f170e02473b682df15bfcdacc3d32d72", + "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72", "shasum": "" }, "require": { @@ -4181,7 +4192,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/exporter/issues", - "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.5" + "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.6" }, "funding": [ { @@ -4189,20 +4200,20 @@ "type": "github" } ], - "time": "2022-09-14T06:03:37+00:00" + "time": "2024-03-02T06:33:00+00:00" }, { "name": "sebastian/global-state", - "version": "5.0.6", + "version": "5.0.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "bde739e7565280bda77be70044ac1047bc007e34" + "reference": "bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bde739e7565280bda77be70044ac1047bc007e34", - "reference": "bde739e7565280bda77be70044ac1047bc007e34", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9", + "reference": "bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9", "shasum": "" }, "require": { @@ -4245,7 +4256,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.6" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.7" }, "funding": [ { @@ -4253,7 +4264,7 @@ "type": "github" } ], - "time": "2023-08-02T09:26:13+00:00" + "time": "2024-03-02T06:35:11+00:00" }, { "name": "sebastian/lines-of-code", @@ -4540,6 +4551,7 @@ "type": "github" } ], + "abandoned": true, "time": "2020-09-28T06:45:17+00:00" }, { @@ -4973,16 +4985,16 @@ }, { "name": "theseer/tokenizer", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/theseer/tokenizer.git", - "reference": "b2ad5003ca10d4ee50a12da31de12a5774ba6b96" + "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b2ad5003ca10d4ee50a12da31de12a5774ba6b96", - "reference": "b2ad5003ca10d4ee50a12da31de12a5774ba6b96", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", + "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", "shasum": "" }, "require": { @@ -5011,7 +5023,7 @@ "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", "support": { "issues": "https://github.com/theseer/tokenizer/issues", - "source": "https://github.com/theseer/tokenizer/tree/1.2.2" + "source": "https://github.com/theseer/tokenizer/tree/1.2.3" }, "funding": [ { @@ -5019,7 +5031,7 @@ "type": "github" } ], - "time": "2023-11-20T00:12:19+00:00" + "time": "2024-03-03T12:36:25+00:00" }, { "name": "twig/twig", From 17d85a478ac199bf3ee7e5d3d740ad6188eb94d9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 14 Mar 2024 01:18:17 +0000 Subject: [PATCH 094/406] fix event test dsn --- tests/unit/Event/EventTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/Event/EventTest.php b/tests/unit/Event/EventTest.php index a84800a28e..d8ce60a35f 100644 --- a/tests/unit/Event/EventTest.php +++ b/tests/unit/Event/EventTest.php @@ -28,7 +28,7 @@ class EventTest extends TestCase 'pass' => App::getEnv('_APP_REDIS_PASS', ''), ]); - $dsn = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); + $dsn = App::getEnv('_APP_CONNECTIONS_QUEUE', 'redis=' . $fallbackForRedis); $dsn = explode('=', $dsn); $dsn = $dsn[1] ?? ''; $dsn = new DSN($dsn); From 24319b841735a5fe2dbe27250602282ec4316dc0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 17 Mar 2024 14:51:27 +0545 Subject: [PATCH 095/406] Fix membership query to use internalId --- app/controllers/api/teams.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 308f19e45e..aa74a49760 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -694,7 +694,7 @@ App::get('/v1/teams/:teamId/memberships') } // Set internal queries - $queries[] = Query::equal('teamId', [$teamId]); + $queries[] = Query::equal('teamInternalId', [$team->getInternalId()]); // Get cursor document if there was a cursor query $cursor = \array_filter($queries, function ($query) { From 4fcb399867b181666532844d9aa79f1d62a8e4b0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 17 Mar 2024 14:54:08 +0545 Subject: [PATCH 096/406] use team internal id for membership check on update --- app/controllers/api/teams.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index aa74a49760..d6a98e5340 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -894,16 +894,16 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') throw new Exception(Exception::MEMBERSHIP_NOT_FOUND); } - if ($membership->getAttribute('teamId') !== $teamId) { - throw new Exception(Exception::TEAM_MEMBERSHIP_MISMATCH); - } - $team = Authorization::skip(fn() => $dbForProject->getDocument('teams', $teamId)); if ($team->isEmpty()) { throw new Exception(Exception::TEAM_NOT_FOUND); } + if ($membership->getAttribute('teamInternalId') !== $team->getInternalId()) { + throw new Exception(Exception::TEAM_MEMBERSHIP_MISMATCH); + } + if (Auth::hash($secret) !== $membership->getAttribute('secret')) { throw new Exception(Exception::TEAM_INVALID_SECRET); } From 05221334e2a195ebd699133a9f81535b1041b7c5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 17 Mar 2024 14:55:31 +0545 Subject: [PATCH 097/406] use team internal id in delete memberships --- app/controllers/api/teams.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index d6a98e5340..978429e51c 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1020,10 +1020,6 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') throw new Exception(Exception::TEAM_INVITE_NOT_FOUND); } - if ($membership->getAttribute('teamId') !== $teamId) { - throw new Exception(Exception::TEAM_MEMBERSHIP_MISMATCH); - } - $user = $dbForProject->getDocument('users', $membership->getAttribute('userId')); if ($user->isEmpty()) { @@ -1036,6 +1032,10 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') throw new Exception(Exception::TEAM_NOT_FOUND); } + if ($membership->getAttribute('teamInternalId') !== $team->getInternalId()) { + throw new Exception(Exception::TEAM_MEMBERSHIP_MISMATCH); + } + $dbForProject->deleteDocument('memberships', $membership->getId()); $dbForProject->deleteCachedDocument('users', $user->getId()); From c5512ae010071d51edd4f6a6837d9769366d20f0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 17 Mar 2024 15:01:09 +0545 Subject: [PATCH 098/406] Update init.php --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 9696b08f6c..cd97fba276 100644 --- a/app/init.php +++ b/app/init.php @@ -1034,7 +1034,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons } if (APP_MODE_ADMIN === $mode) { - if ($user->find('teamId', $project->getAttribute('teamId'), 'memberships')) { + if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) { Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. } else { $user = new Document([]); From c849ff0b0f642c9e444e3e69f958485f4c63c992 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 18 Mar 2024 08:52:47 +0000 Subject: [PATCH 099/406] use internal ids for query --- app/controllers/api/account.php | 4 ++-- app/controllers/api/users.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 345486b657..3df4d47355 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -555,7 +555,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $identityWithMatchingEmail = $dbForProject->findOne('identities', [ Query::equal('providerEmail', [$email]), - Query::notEqual('userId', $userId), + Query::notEqual('userInternalId', $user->getInternalId()), ]); if (!empty($identityWithMatchingEmail)) { throw new Exception(Exception::USER_ALREADY_EXISTS); @@ -1985,7 +1985,7 @@ App::patch('/v1/account/email') // Makes sure this email is not already used in another identity $identityWithMatchingEmail = $dbForProject->findOne('identities', [ Query::equal('providerEmail', [$email]), - Query::notEqual('userId', $user->getId()), + Query::notEqual('userInternalId', $user->getInternalId()), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 5ce2263f47..6a0d6f3d7a 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -941,7 +941,7 @@ App::patch('/v1/users/:userId/email') // Makes sure this email is not already used in another identity $identityWithMatchingEmail = $dbForProject->findOne('identities', [ Query::equal('providerEmail', [$email]), - Query::notEqual('userId', $user->getId()), + Query::notEqual('userInternalId', $user->getInternalId()), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); From 21d4b8feef5b28faf11f042187ddfb1f3b5d9d4d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 18 Mar 2024 09:01:54 +0000 Subject: [PATCH 100/406] check user internal Id on membership --- app/controllers/api/teams.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 978429e51c..03f158c131 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -916,7 +916,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') $user->setAttributes($dbForProject->getDocument('users', $userId)->getArrayCopy()); // Get user } - if ($membership->getAttribute('userId') !== $user->getId()) { + if ($membership->getAttribute('userInternalId') !== $user->getInternalId()) { throw new Exception(Exception::TEAM_INVITE_MISMATCH, 'Invite does not belong to current user (' . $user->getAttribute('email') . ')'); } From a306cb87d88c16789cca1fab9c6eece2520d4b2f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 18 Mar 2024 12:07:14 +0000 Subject: [PATCH 101/406] chore: update avatars API --- app/controllers/api/avatars.php | 61 ++++++++++++++--------------- composer.json | 1 + composer.lock | 69 +++++++++++++++++++++++++-------- 3 files changed, 84 insertions(+), 47 deletions(-) diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index e0d967eb00..b9df4b6345 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -23,6 +23,7 @@ use Utopia\Validator\URL; use Utopia\Validator\WhiteList; use chillerlan\QRCode\QRCode; use chillerlan\QRCode\QROptions; +use Utopia\Fetch\Client; $avatarCallback = function (string $type, string $code, int $width, int $height, int $quality, Response $response) { @@ -283,14 +284,17 @@ App::get('/v1/avatars/image') throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } - $fetch = @\file_get_contents($url); + $client = new Client(); + $res = $client + ->setAllowRedirects(false) + ->fetch($url); - if (!$fetch) { + if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_IMAGE_NOT_FOUND); } try { - $image = new Image($fetch); + $image = new Image($res->getBody()); } catch (\Exception $exception) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to parse image'); } @@ -339,31 +343,23 @@ App::get('/v1/avatars/favicon') throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } - $curl = \curl_init(); + $client = new Client(); + $res = $client + ->setAllowRedirects(false) + ->setUserAgent(\sprintf( + APP_USERAGENT, + App::getEnv('_APP_VERSION', 'UNKNOWN'), + App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY) + )) + ->fetch($url); - \curl_setopt_array($curl, [ - CURLOPT_RETURNTRANSFER => 1, - CURLOPT_FOLLOWLOCATION => true, - CURLOPT_MAXREDIRS => 3, - CURLOPT_URL => $url, - CURLOPT_USERAGENT => \sprintf( - APP_USERAGENT, - App::getEnv('_APP_VERSION', 'UNKNOWN'), - App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY) - ), - ]); - - $html = \curl_exec($curl); - - \curl_close($curl); - - if (!$html) { + if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } $doc = new DOMDocument(); $doc->strictErrorChecking = false; - @$doc->loadHTML($html); + @$doc->loadHTML($res->getBody()); $links = $doc->getElementsByTagName('link'); $outputHref = ''; @@ -418,9 +414,18 @@ App::get('/v1/avatars/favicon') throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } - if ('ico' == $outputExt) { // Skip crop, Imagick isn\'t supporting icon files - $data = @\file_get_contents($outputHref, false); + $client = new Client(); + $res = $client + ->setAllowRedirects(false) + ->fetch($outputHref); + if ($res->getStatusCode() !== 200) { + throw new Exception(Exception::AVATAR_ICON_NOT_FOUND); + } + + $data = $res->getBody(); + + if ('ico' == $outputExt) { // Skip crop, Imagick isn\'t supporting icon files if (empty($data) || (\mb_substr($data, 0, 5) === 'file($data); } - $fetch = @\file_get_contents($outputHref, false); - - if (!$fetch) { - throw new Exception(Exception::AVATAR_ICON_NOT_FOUND); - } - - $image = new Image($fetch); + $image = new Image($data); $image->crop((int) $width, (int) $height); $output = (empty($output)) ? $type : $output; $data = $image->output($output, $quality); diff --git a/composer.json b/composer.json index faf30a117d..eaa6cfa206 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,7 @@ "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.1.*", "utopia-php/framework": "0.33.*", + "utopia-php/fetch": "0.2.*", "utopia-php/image": "0.6.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", diff --git a/composer.lock b/composer.lock index c7fd4959b5..606338c675 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9492cb777590ff776c2a73620783d74b", + "content-hash": "95d4c9ccd4b2f958ca247d83b04d1391", "packages": [ { "name": "adhocore/jwt", @@ -1352,6 +1352,45 @@ }, "time": "2022-10-26T10:06:20+00:00" }, + { + "name": "utopia-php/fetch", + "version": "0.2.1", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "1423c0ee3eef944d816ca6e31706895b585aea82" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/1423c0ee3eef944d816ca6e31706895b585aea82", + "reference": "1423c0ee3eef944d816ca6e31706895b585aea82", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.2.1" + }, + "time": "2024-03-18T11:50:59+00:00" + }, { "name": "utopia-php/framework", "version": "0.33.3", @@ -2585,16 +2624,16 @@ }, { "name": "matthiasmullie/minify", - "version": "1.3.72", + "version": "1.3.73", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "531fdeef1911ffe27a53f8a19c297648c78f757e" + "reference": "cb7a9297b4ab070909cefade30ee95054d4ae87a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/531fdeef1911ffe27a53f8a19c297648c78f757e", - "reference": "531fdeef1911ffe27a53f8a19c297648c78f757e", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/cb7a9297b4ab070909cefade30ee95054d4ae87a", + "reference": "cb7a9297b4ab070909cefade30ee95054d4ae87a", "shasum": "" }, "require": { @@ -2644,7 +2683,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.72" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.73" }, "funding": [ { @@ -2652,7 +2691,7 @@ "type": "github" } ], - "time": "2024-03-13T12:02:00+00:00" + "time": "2024-03-15T10:27:10+00:00" }, { "name": "matthiasmullie/path-converter", @@ -4500,16 +4539,16 @@ }, { "name": "sebastian/resource-operations", - "version": "3.0.3", + "version": "3.0.4", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/resource-operations.git", - "reference": "0f4443cb3a1d92ce809899753bc0d5d5a8dd19a8" + "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/0f4443cb3a1d92ce809899753bc0d5d5a8dd19a8", - "reference": "0f4443cb3a1d92ce809899753bc0d5d5a8dd19a8", + "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/05d5692a7993ecccd56a03e40cd7e5b09b1d404e", + "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e", "shasum": "" }, "require": { @@ -4521,7 +4560,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "3.0-dev" + "dev-main": "3.0-dev" } }, "autoload": { @@ -4542,8 +4581,7 @@ "description": "Provides a list of PHP built-in functions that operate on resources", "homepage": "https://www.github.com/sebastianbergmann/resource-operations", "support": { - "issues": "https://github.com/sebastianbergmann/resource-operations/issues", - "source": "https://github.com/sebastianbergmann/resource-operations/tree/3.0.3" + "source": "https://github.com/sebastianbergmann/resource-operations/tree/3.0.4" }, "funding": [ { @@ -4551,8 +4589,7 @@ "type": "github" } ], - "abandoned": true, - "time": "2020-09-28T06:45:17+00:00" + "time": "2024-03-14T16:00:52+00:00" }, { "name": "sebastian/type", From 5e6c94d30720714c819b506de5a289e7584d8b29 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 18 Mar 2024 12:41:51 +0000 Subject: [PATCH 102/406] chore: update checks --- app/controllers/api/avatars.php | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index b9df4b6345..734cfb2dc5 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -285,10 +285,14 @@ App::get('/v1/avatars/image') } $client = new Client(); - $res = $client + try { + $res = $client ->setAllowRedirects(false) ->fetch($url); - + } catch (\Throwable) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); + } + if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_IMAGE_NOT_FOUND); } @@ -344,7 +348,8 @@ App::get('/v1/avatars/favicon') } $client = new Client(); - $res = $client + try { + $res = $client ->setAllowRedirects(false) ->setUserAgent(\sprintf( APP_USERAGENT, @@ -352,7 +357,10 @@ App::get('/v1/avatars/favicon') App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY) )) ->fetch($url); - + } catch (\Throwable) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); + } + if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } @@ -415,9 +423,13 @@ App::get('/v1/avatars/favicon') } $client = new Client(); - $res = $client - ->setAllowRedirects(false) - ->fetch($outputHref); + try { + $res = $client + ->setAllowRedirects(false) + ->fetch($outputHref); + } catch (\Throwable) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); + } if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_ICON_NOT_FOUND); From eeba452fb536032b4dc467cd08e4cc7be338b5dd Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 18 Mar 2024 12:43:42 +0000 Subject: [PATCH 103/406] chore: linter --- app/controllers/api/avatars.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/avatars.php b/app/controllers/api/avatars.php index 734cfb2dc5..d0f8072f2c 100644 --- a/app/controllers/api/avatars.php +++ b/app/controllers/api/avatars.php @@ -292,7 +292,7 @@ App::get('/v1/avatars/image') } catch (\Throwable) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } - + if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_IMAGE_NOT_FOUND); } @@ -360,7 +360,7 @@ App::get('/v1/avatars/favicon') } catch (\Throwable) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } - + if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } @@ -429,7 +429,7 @@ App::get('/v1/avatars/favicon') ->fetch($outputHref); } catch (\Throwable) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); - } + } if ($res->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_ICON_NOT_FOUND); From 63a30ed0831655b34ba329a731183d7192d7aa19 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 19 Mar 2024 12:15:39 +0200 Subject: [PATCH 104/406] maximumFileSize fix --- app/controllers/api/storage.php | 4 +++- app/init.php | 3 +++ composer.lock | 17 +++++++++-------- docker-compose.yml | 1 + 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 26ae0adacf..4ee330c320 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -2,9 +2,11 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Hash\Sha; +use Appwrite\Auth\Validator\PasswordDictionary; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Hooks\Hooks; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Response; @@ -65,7 +67,7 @@ App::post('/v1/storage/buckets') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) diff --git a/app/init.php b/app/init.php index cd97fba276..ec7fa7163b 100644 --- a/app/init.php +++ b/app/init.php @@ -1494,3 +1494,6 @@ App::setResource('requestTimestamp', function ($request) { } return $requestTimestamp; }, ['request']); +App::setResource('plan', function (array $plan = []) { + return []; +}); diff --git a/composer.lock b/composer.lock index 606338c675..0d7d13460a 100644 --- a/composer.lock +++ b/composer.lock @@ -1393,16 +1393,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.3", + "version": "0.33.4", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "893d602cd96676810c25fc9b9a2e9360eebb898f" + "reference": "6ee2e93a4a529bce5eecf4ac892890d4b0c84861" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/893d602cd96676810c25fc9b9a2e9360eebb898f", - "reference": "893d602cd96676810c25fc9b9a2e9360eebb898f", + "url": "https://api.github.com/repos/utopia-php/http/zipball/6ee2e93a4a529bce5eecf4ac892890d4b0c84861", + "reference": "6ee2e93a4a529bce5eecf4ac892890d4b0c84861", "shasum": "" }, "require": { @@ -1417,7 +1417,8 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/" + "Utopia\\": "src/", + "Tests\\E2E\\": "tests/e2e" } }, "notification-url": "https://packagist.org/downloads/", @@ -1432,9 +1433,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.3" + "source": "https://github.com/utopia-php/http/tree/0.33.4" }, - "time": "2024-03-11T13:43:23+00:00" + "time": "2024-03-19T08:04:59+00:00" }, { "name": "utopia-php/image", @@ -5169,5 +5170,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 26c5301526..da60452513 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev + - ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework depends_on: - mariadb - redis From 2213ddb54da99577e6efdd804c8d802d27b725cd Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 19 Mar 2024 12:54:47 +0200 Subject: [PATCH 105/406] maximumFileSize fix --- app/controllers/api/storage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 4ee330c320..06b407bbfc 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -67,7 +67,7 @@ App::post('/v1/storage/buckets') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) From 0ee4b956482b1edd1f69c95c4a4723d2e9786819 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 19 Mar 2024 15:09:03 +0200 Subject: [PATCH 106/406] maximumFileSize fix --- app/controllers/api/storage.php | 2 +- docker-compose.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 06b407bbfc..6edca33604 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -67,7 +67,7 @@ App::post('/v1/storage/buckets') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true) - ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) + ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) App::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1024 * 1024 * 1024), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan']) ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true) ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true) ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true) diff --git a/docker-compose.yml b/docker-compose.yml index da60452513..26c5301526 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev - - ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework depends_on: - mariadb - redis From fa97ac7019e83e982cdad9af49aec74551fec38c Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 19 Mar 2024 15:16:21 +0200 Subject: [PATCH 107/406] maximumFileSize fix --- app/controllers/api/storage.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 6edca33604..a0a017db45 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1,12 +1,9 @@ Date: Wed, 20 Mar 2024 13:47:20 +0100 Subject: [PATCH 108/406] Make realtime resources configurable --- app/realtime.php | 82 ++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index f7fc7070a4..652b6d9a28 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -36,54 +36,60 @@ require_once __DIR__ . '/init.php'; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); -function getConsoleDB(): Database -{ - global $register; +// Allows overriding +if(!function_exists("getConsoleDB")) { + function getConsoleDB(): Database + { + global $register; - /** @var \Utopia\Pools\Group $pools */ - $pools = $register->get('pools'); + /** @var \Utopia\Pools\Group $pools */ + $pools = $register->get('pools'); - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource() - ; + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; - $database = new Database($dbAdapter, getCache()); + $database = new Database($dbAdapter, getCache()); - $database - ->setNamespace('_console') - ->setMetadata('host', \gethostname()) - ->setMetadata('project', '_console'); + $database + ->setNamespace('_console') + ->setMetadata('host', \gethostname()) + ->setMetadata('project', '_console'); - return $database; + return $database; + } } -function getProjectDB(Document $project): Database -{ - global $register; +// Allows overriding +if(!function_exists("getProjectDB")) { + function getProjectDB(Document $project): Database + { + global $register; - /** @var \Utopia\Pools\Group $pools */ - $pools = $register->get('pools'); + /** @var \Utopia\Pools\Group $pools */ + $pools = $register->get('pools'); - if ($project->isEmpty() || $project->getId() === 'console') { - return getConsoleDB(); + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database + ->setNamespace('_' . $project->getInternalId()) + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()); + + return $database; } - - $dbAdapter = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; - - $database = new Database($dbAdapter, getCache()); - - $database - ->setNamespace('_' . $project->getInternalId()) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()); - - return $database; } function getCache(): Cache From 440d924518b1d237dc3db6024372af77b732d1be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 20 Mar 2024 14:14:23 +0100 Subject: [PATCH 109/406] Allow getCache override --- app/realtime.php | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 652b6d9a28..c97eb598b8 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -92,24 +92,27 @@ if(!function_exists("getProjectDB")) { } } -function getCache(): Cache -{ - global $register; +// Allows overriding +if(!function_exists("getCache")) { + function getCache(): Cache + { + global $register; - $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ - $list = Config::getParam('pools-cache', []); - $adapters = []; + $list = Config::getParam('pools-cache', []); + $adapters = []; - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource() - ; + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); } - - return new Cache(new Sharding($adapters)); } $realtime = new Realtime(); From fcfff2ab048f03eac060f86a542f568595598acb Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:46:42 +0000 Subject: [PATCH 110/406] chore: remove dead code in error template --- app/views/general/error.phtml | 39 ----------------------------------- 1 file changed, 39 deletions(-) diff --git a/app/views/general/error.phtml b/app/views/general/error.phtml index 450dcf8973..6c75ee1e5f 100644 --- a/app/views/general/error.phtml +++ b/app/views/general/error.phtml @@ -111,45 +111,6 @@ $title = $this->getParam('title', '') -