From 6a9eeeb370f4dfb2b4ac7757dafc45b23f124e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 19:39:24 +0100 Subject: [PATCH 1/7] Add createFunction abuse labels --- app/controllers/api/functions.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 003efbad03..25e8886a46 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -141,6 +141,9 @@ App::post('/v1/functions') ->label('resourceType', RESOURCE_TYPE_FUNCTIONS) ->label('audits.event', 'function.create') ->label('audits.resource', 'function/{response.$id}') + ->label('abuse-key', 'projectId:{projectId}') + ->label('abuse-limit', 50) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT * 24) // 1 day ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'create') From a18472507b572dcf704b1929b88d8663ddb86653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:45:49 +0100 Subject: [PATCH 2/7] Add createFunction abuse check + fix abuse --- app/controllers/api/functions.php | 31 ++++++++++++++++++++++++++++--- composer.lock | 14 +++++++------- docker-compose.yml | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 25e8886a46..b263535777 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -23,6 +23,8 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model\Rule; use Executor\Executor; use MaxMind\Db\Reader; +use Utopia\Abuse\Abuse; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -141,9 +143,6 @@ App::post('/v1/functions') ->label('resourceType', RESOURCE_TYPE_FUNCTIONS) ->label('audits.event', 'function.create') ->label('audits.resource', 'function/{response.$id}') - ->label('abuse-key', 'projectId:{projectId}') - ->label('abuse-limit', 50) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT * 24) // 1 day ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'create') @@ -190,6 +189,32 @@ App::post('/v1/functions') ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; + // Temporary abuse check + $abuseKey = "projectId:{projectId},url:{url}"; + $abuseLimit = 5; + $abuseTime = 86400; // 1 day + + $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); + $timeLimit + ->setParam('{projectId}', $project->getId()) + ->setParam('{url}', '/v1/functions'); + + $abuse = new Abuse($timeLimit); + $remaining = $timeLimit->remaining(); + $limit = $timeLimit->limit(); + $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $abuseTime; + + $response + ->addHeader('X-RateLimit-Limit', $limit) + ->addHeader('X-RateLimit-Remaining', $remaining) + ->addHeader('X-RateLimit-Reset', $time); + + $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; + if($enabled && $abuse->check()) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); + } + // End of temporary abuse check + $allowList = \array_filter(\explode(',', System::getEnv('_APP_FUNCTIONS_RUNTIMES', ''))); if (!empty($allowList) && !\in_array($runtime, $allowList)) { diff --git a/composer.lock b/composer.lock index ee947d27f0..732bafd219 100644 --- a/composer.lock +++ b/composer.lock @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.1", + "version": "0.43.2", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa" + "reference": "374536b86d8d39066960a7da161d444a099bbc56" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/e404c21e8dcf6a310bc83cf1d74e716b105598fa", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/374536b86d8d39066960a7da161d444a099bbc56", + "reference": "374536b86d8d39066960a7da161d444a099bbc56", "shasum": "" }, "require": { @@ -3153,7 +3153,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.53.*" + "utopia-php/database": "0.53.200" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.1" + "source": "https://github.com/utopia-php/abuse/tree/0.43.2" }, - "time": "2024-10-23T04:29:12+00:00" + "time": "2024-12-12T19:43:24+00:00" }, { "name": "utopia-php/analytics", diff --git a/docker-compose.yml b/docker-compose.yml index f9a79ee84d..db5416d2b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,6 +83,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev + - ./vendor/utopia-php/abuse:/usr/src/code/vendor/utopia-php/abuse depends_on: - mariadb - redis From 9c4d0f2a79a6f6497c1761f9017f8780be19fdf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:48:28 +0100 Subject: [PATCH 3/7] Leftover --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index db5416d2b5..f9a79ee84d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,7 +83,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev - - ./vendor/utopia-php/abuse:/usr/src/code/vendor/utopia-php/abuse depends_on: - mariadb - redis From 04139d690181f6f4bb08883005ad8b8d756d2cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:52:31 +0100 Subject: [PATCH 4/7] Linter fix --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b263535777..76916f2770 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -210,7 +210,7 @@ App::post('/v1/functions') ->addHeader('X-RateLimit-Reset', $time); $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; - if($enabled && $abuse->check()) { + if ($enabled && $abuse->check()) { throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); } // End of temporary abuse check From ab7ea056e641b42153d9bc41260903bddea492ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 21:00:04 +0100 Subject: [PATCH 5/7] Increase limit --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 76916f2770..395069bc02 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -191,7 +191,7 @@ App::post('/v1/functions') // Temporary abuse check $abuseKey = "projectId:{projectId},url:{url}"; - $abuseLimit = 5; + $abuseLimit = 50; $abuseTime = 86400; // 1 day $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); From 8b3f09a981b320a180c5946e04b97aca2b67e257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 21:10:22 +0100 Subject: [PATCH 6/7] Env var for function abuse limit --- .env | 1 + app/controllers/api/functions.php | 2 +- docker-compose.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 2470c786b1..8f7d7996e7 100644 --- a/.env +++ b/.env @@ -109,3 +109,4 @@ _APP_MESSAGE_EMAIL_TEST_DSN= _APP_MESSAGE_PUSH_TEST_DSN= _APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 _APP_PROJECT_REGIONS=default +_APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000 diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 395069bc02..d473f60a0b 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -191,7 +191,7 @@ App::post('/v1/functions') // Temporary abuse check $abuseKey = "projectId:{projectId},url:{url}"; - $abuseLimit = 50; + $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); $abuseTime = 86400; // 1 day $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); diff --git a/docker-compose.yml b/docker-compose.yml index f9a79ee84d..e78af8b307 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -195,6 +195,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_DATABASE_SHARED_NAMESPACE + - _APP_FUNCTIONS_CREATION_ABUSE_LIMIT appwrite-console: <<: *x-logging From 905b4938a495e22b481170049b37c7f3d205d9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:55:33 +0000 Subject: [PATCH 7/7] Fix failing test --- app/controllers/api/functions.php | 43 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d473f60a0b..5934258037 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -190,30 +190,33 @@ App::post('/v1/functions') $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check - $abuseKey = "projectId:{projectId},url:{url}"; - $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); - $abuseTime = 86400; // 1 day + $abuseCheck = function () use ($project, $dbForProject, $response) { + $abuseKey = "projectId:{projectId},url:{url}"; + $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); + $abuseTime = 86400; // 1 day - $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); - $timeLimit - ->setParam('{projectId}', $project->getId()) - ->setParam('{url}', '/v1/functions'); + $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); + $timeLimit + ->setParam('{projectId}', $project->getId()) + ->setParam('{url}', '/v1/functions'); - $abuse = new Abuse($timeLimit); - $remaining = $timeLimit->remaining(); - $limit = $timeLimit->limit(); - $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $abuseTime; + $abuse = new Abuse($timeLimit); + $remaining = $timeLimit->remaining(); + $limit = $timeLimit->limit(); + $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $abuseTime; - $response - ->addHeader('X-RateLimit-Limit', $limit) - ->addHeader('X-RateLimit-Remaining', $remaining) - ->addHeader('X-RateLimit-Reset', $time); + $response + ->addHeader('X-RateLimit-Limit', $limit) + ->addHeader('X-RateLimit-Remaining', $remaining) + ->addHeader('X-RateLimit-Reset', $time); - $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; - if ($enabled && $abuse->check()) { - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); - } - // End of temporary abuse check + $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; + if ($enabled && $abuse->check()) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); + } + }; + + $abuseCheck(); $allowList = \array_filter(\explode(',', System::getEnv('_APP_FUNCTIONS_RUNTIMES', '')));