From a12b758658e32d01df7e1a2c5fa54b58b5855bce Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:11:22 +0000 Subject: [PATCH 01/21] feat: upgrade assistant to 0.6,0 --- app/controllers/api/console.php | 8 ++++++-- docker-compose.yml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/console.php b/app/controllers/api/console.php index eeb823a3d3..b7c7c05641 100644 --- a/app/controllers/api/console.php +++ b/app/controllers/api/console.php @@ -69,11 +69,15 @@ App::post('/v1/console/assistant') ->label('abuse-limit', 15) ->label('abuse-key', 'userId:{userId}') ->param('prompt', '', new Text(2000), 'Prompt. A string containing questions asked to the AI assistant.') + ->param('systemPrompt', '', new Text(2000), 'System Prompt. A string containing context to help the AI assistant provide better responses.') ->inject('response') - ->action(function (string $prompt, Response $response) { + ->action(function (string $prompt, string $systemPrompt, Response $response) { $ch = curl_init('http://appwrite-assistant:3003/'); $responseHeaders = []; - $query = json_encode(['prompt' => $prompt]); + $query = json_encode([ + 'prompt' => $prompt, + 'systemPrompt' => $systemPrompt, + ]); $headers = ['accept: text/event-stream']; $handleEvent = function ($ch, $data) use ($response) { $response->chunk($data); diff --git a/docker-compose.yml b/docker-compose.yml index f9a79ee84d..9652767a1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -868,7 +868,7 @@ services: appwrite-assistant: container_name: appwrite-assistant - image: appwrite/assistant:0.5.0 + image: appwrite/assistant:0.6.0 networks: - appwrite environment: From 717b06950ec32e12928c4e2ceb80e8e5d400dbec Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 12 Dec 2024 20:20:47 +0200 Subject: [PATCH 02/21] Usage payload debug --- src/Appwrite/Platform/Workers/Usage.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 98b2ea32c7..855f9c1ebc 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -60,12 +60,13 @@ class Usage extends Action } $document = $payload['project'] ?? []; + $project = new Document($document); - if (empty($document)) { + if (empty($project->getAttribute('database'))) { + var_dump(1); var_dump($payload); } - $project = new Document($document); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { if (empty($document)) { @@ -80,6 +81,11 @@ class Usage extends Action ); } + if (empty($project->getAttribute('database'))) { + var_dump(2); + var_dump($payload); + } + $this->stats[$projectId]['project'] = [ '$id' => $project->getId(), '$internalId' => $project->getInternalId(), From 6a9eeeb370f4dfb2b4ac7757dafc45b23f124e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 19:39:24 +0100 Subject: [PATCH 03/21] Add createFunction abuse labels --- app/controllers/api/functions.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 003efbad03..25e8886a46 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -141,6 +141,9 @@ App::post('/v1/functions') ->label('resourceType', RESOURCE_TYPE_FUNCTIONS) ->label('audits.event', 'function.create') ->label('audits.resource', 'function/{response.$id}') + ->label('abuse-key', 'projectId:{projectId}') + ->label('abuse-limit', 50) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT * 24) // 1 day ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'create') From bc38fd26407931d0a6a34c394569039830cc718d Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:04:31 +0000 Subject: [PATCH 04/21] fix: hide system prompt --- app/controllers/api/console.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/console.php b/app/controllers/api/console.php index b7c7c05641..eeb823a3d3 100644 --- a/app/controllers/api/console.php +++ b/app/controllers/api/console.php @@ -69,15 +69,11 @@ App::post('/v1/console/assistant') ->label('abuse-limit', 15) ->label('abuse-key', 'userId:{userId}') ->param('prompt', '', new Text(2000), 'Prompt. A string containing questions asked to the AI assistant.') - ->param('systemPrompt', '', new Text(2000), 'System Prompt. A string containing context to help the AI assistant provide better responses.') ->inject('response') - ->action(function (string $prompt, string $systemPrompt, Response $response) { + ->action(function (string $prompt, Response $response) { $ch = curl_init('http://appwrite-assistant:3003/'); $responseHeaders = []; - $query = json_encode([ - 'prompt' => $prompt, - 'systemPrompt' => $systemPrompt, - ]); + $query = json_encode(['prompt' => $prompt]); $headers = ['accept: text/event-stream']; $handleEvent = function ($ch, $data) use ($response) { $response->chunk($data); From 177fd92094d68b2f07b7fc884a882f7514c23a92 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 12 Dec 2024 21:20:55 +0200 Subject: [PATCH 05/21] Usage payload debug --- src/Appwrite/Platform/Workers/Usage.php | 6 +----- src/Appwrite/Platform/Workers/UsageDump.php | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 855f9c1ebc..8199fe73a7 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -63,8 +63,8 @@ class Usage extends Action $project = new Document($document); if (empty($project->getAttribute('database'))) { - var_dump(1); var_dump($payload); + return; } $projectId = $project->getInternalId(); @@ -81,10 +81,6 @@ class Usage extends Action ); } - if (empty($project->getAttribute('database'))) { - var_dump(2); - var_dump($payload); - } $this->stats[$projectId]['project'] = [ '$id' => $project->getId(), diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 1fe6b742b8..78235ce7e9 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -71,10 +71,6 @@ class UsageDump extends Action $project = new Document($document); - if (empty($project->getAttribute('database'))) { - var_dump($stats); - } - /** * End temp bug fallback */ From a18472507b572dcf704b1929b88d8663ddb86653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:45:49 +0100 Subject: [PATCH 06/21] Add createFunction abuse check + fix abuse --- app/controllers/api/functions.php | 31 ++++++++++++++++++++++++++++--- composer.lock | 14 +++++++------- docker-compose.yml | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 25e8886a46..b263535777 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -23,6 +23,8 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model\Rule; use Executor\Executor; use MaxMind\Db\Reader; +use Utopia\Abuse\Abuse; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -141,9 +143,6 @@ App::post('/v1/functions') ->label('resourceType', RESOURCE_TYPE_FUNCTIONS) ->label('audits.event', 'function.create') ->label('audits.resource', 'function/{response.$id}') - ->label('abuse-key', 'projectId:{projectId}') - ->label('abuse-limit', 50) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT * 24) // 1 day ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'create') @@ -190,6 +189,32 @@ App::post('/v1/functions') ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; + // Temporary abuse check + $abuseKey = "projectId:{projectId},url:{url}"; + $abuseLimit = 5; + $abuseTime = 86400; // 1 day + + $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); + $timeLimit + ->setParam('{projectId}', $project->getId()) + ->setParam('{url}', '/v1/functions'); + + $abuse = new Abuse($timeLimit); + $remaining = $timeLimit->remaining(); + $limit = $timeLimit->limit(); + $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $abuseTime; + + $response + ->addHeader('X-RateLimit-Limit', $limit) + ->addHeader('X-RateLimit-Remaining', $remaining) + ->addHeader('X-RateLimit-Reset', $time); + + $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; + if($enabled && $abuse->check()) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); + } + // End of temporary abuse check + $allowList = \array_filter(\explode(',', System::getEnv('_APP_FUNCTIONS_RUNTIMES', ''))); if (!empty($allowList) && !\in_array($runtime, $allowList)) { diff --git a/composer.lock b/composer.lock index ee947d27f0..732bafd219 100644 --- a/composer.lock +++ b/composer.lock @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.1", + "version": "0.43.2", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa" + "reference": "374536b86d8d39066960a7da161d444a099bbc56" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/e404c21e8dcf6a310bc83cf1d74e716b105598fa", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/374536b86d8d39066960a7da161d444a099bbc56", + "reference": "374536b86d8d39066960a7da161d444a099bbc56", "shasum": "" }, "require": { @@ -3153,7 +3153,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.53.*" + "utopia-php/database": "0.53.200" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.1" + "source": "https://github.com/utopia-php/abuse/tree/0.43.2" }, - "time": "2024-10-23T04:29:12+00:00" + "time": "2024-12-12T19:43:24+00:00" }, { "name": "utopia-php/analytics", diff --git a/docker-compose.yml b/docker-compose.yml index f9a79ee84d..db5416d2b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,6 +83,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev + - ./vendor/utopia-php/abuse:/usr/src/code/vendor/utopia-php/abuse depends_on: - mariadb - redis From 9c4d0f2a79a6f6497c1761f9017f8780be19fdf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:48:28 +0100 Subject: [PATCH 07/21] Leftover --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index db5416d2b5..f9a79ee84d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,7 +83,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev - - ./vendor/utopia-php/abuse:/usr/src/code/vendor/utopia-php/abuse depends_on: - mariadb - redis From 04139d690181f6f4bb08883005ad8b8d756d2cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:52:31 +0100 Subject: [PATCH 08/21] Linter fix --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b263535777..76916f2770 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -210,7 +210,7 @@ App::post('/v1/functions') ->addHeader('X-RateLimit-Reset', $time); $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; - if($enabled && $abuse->check()) { + if ($enabled && $abuse->check()) { throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); } // End of temporary abuse check From 01019f505541897b07a4246385887ca225b4de11 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 12 Dec 2024 21:54:59 +0200 Subject: [PATCH 09/21] Usage payload debug --- src/Appwrite/Platform/Workers/UsageDump.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 78235ce7e9..3e50ba0363 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -71,6 +71,10 @@ class UsageDump extends Action $project = new Document($document); + if (empty($project->getAttribute('database'))) { + continue; + } + /** * End temp bug fallback */ From ab7ea056e641b42153d9bc41260903bddea492ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 21:00:04 +0100 Subject: [PATCH 10/21] Increase limit --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 76916f2770..395069bc02 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -191,7 +191,7 @@ App::post('/v1/functions') // Temporary abuse check $abuseKey = "projectId:{projectId},url:{url}"; - $abuseLimit = 5; + $abuseLimit = 50; $abuseTime = 86400; // 1 day $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); From 8b3f09a981b320a180c5946e04b97aca2b67e257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 21:10:22 +0100 Subject: [PATCH 11/21] Env var for function abuse limit --- .env | 1 + app/controllers/api/functions.php | 2 +- docker-compose.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 2470c786b1..8f7d7996e7 100644 --- a/.env +++ b/.env @@ -109,3 +109,4 @@ _APP_MESSAGE_EMAIL_TEST_DSN= _APP_MESSAGE_PUSH_TEST_DSN= _APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 _APP_PROJECT_REGIONS=default +_APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000 diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 395069bc02..d473f60a0b 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -191,7 +191,7 @@ App::post('/v1/functions') // Temporary abuse check $abuseKey = "projectId:{projectId},url:{url}"; - $abuseLimit = 50; + $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); $abuseTime = 86400; // 1 day $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); diff --git a/docker-compose.yml b/docker-compose.yml index f9a79ee84d..e78af8b307 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -195,6 +195,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_DATABASE_SHARED_NAMESPACE + - _APP_FUNCTIONS_CREATION_ABUSE_LIMIT appwrite-console: <<: *x-logging From 905b4938a495e22b481170049b37c7f3d205d9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 12 Dec 2024 20:55:33 +0000 Subject: [PATCH 12/21] Fix failing test --- app/controllers/api/functions.php | 43 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d473f60a0b..5934258037 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -190,30 +190,33 @@ App::post('/v1/functions') $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check - $abuseKey = "projectId:{projectId},url:{url}"; - $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); - $abuseTime = 86400; // 1 day + $abuseCheck = function () use ($project, $dbForProject, $response) { + $abuseKey = "projectId:{projectId},url:{url}"; + $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); + $abuseTime = 86400; // 1 day - $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); - $timeLimit - ->setParam('{projectId}', $project->getId()) - ->setParam('{url}', '/v1/functions'); + $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); + $timeLimit + ->setParam('{projectId}', $project->getId()) + ->setParam('{url}', '/v1/functions'); - $abuse = new Abuse($timeLimit); - $remaining = $timeLimit->remaining(); - $limit = $timeLimit->limit(); - $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $abuseTime; + $abuse = new Abuse($timeLimit); + $remaining = $timeLimit->remaining(); + $limit = $timeLimit->limit(); + $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $abuseTime; - $response - ->addHeader('X-RateLimit-Limit', $limit) - ->addHeader('X-RateLimit-Remaining', $remaining) - ->addHeader('X-RateLimit-Reset', $time); + $response + ->addHeader('X-RateLimit-Limit', $limit) + ->addHeader('X-RateLimit-Remaining', $remaining) + ->addHeader('X-RateLimit-Reset', $time); - $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; - if ($enabled && $abuse->check()) { - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); - } - // End of temporary abuse check + $enabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; + if ($enabled && $abuse->check()) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); + } + }; + + $abuseCheck(); $allowList = \array_filter(\explode(',', System::getEnv('_APP_FUNCTIONS_RUNTIMES', ''))); From 515b5904b849136ff755a4c936cb328c1834e4f6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 10 Dec 2024 11:34:51 +0000 Subject: [PATCH 13/21] update project accessed at from router --- app/controllers/general.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 32cc68fecf..2c43116c10 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -91,6 +91,15 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $project = Authorization::skip( fn () => $dbForPlatform->getDocument('projects', $projectId) ); + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } + if (array_key_exists('proxy', $project->getAttribute('services', []))) { $status = $project->getAttribute('services', [])['proxy']; if (!$status) { From ab4a4441545ac30517610797e1adc0274103f195 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 10 Dec 2024 11:13:09 +0000 Subject: [PATCH 14/21] Fix: project accessed at on workers --- app/worker.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/worker.php b/app/worker.php index 2c7d8acb22..6fdb49d35a 100644 --- a/app/worker.php +++ b/app/worker.php @@ -374,6 +374,20 @@ try { $worker = $platform->getWorker(); +$worker + ->init() + ->inject('project') + ->inject('dbForConsole') + ->action(function (Document $project, Database $dbForConsole) { + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } + }); + $worker ->shutdown() ->inject('pools') From a24dc7903648671e0a3d515dde9bd9cec85e4581 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:36:02 +0000 Subject: [PATCH 15/21] chore: fix endpoint --- app/controllers/api/console.php | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/console.php b/app/controllers/api/console.php index eeb823a3d3..1b307b1d14 100644 --- a/app/controllers/api/console.php +++ b/app/controllers/api/console.php @@ -71,7 +71,7 @@ App::post('/v1/console/assistant') ->param('prompt', '', new Text(2000), 'Prompt. A string containing questions asked to the AI assistant.') ->inject('response') ->action(function (string $prompt, Response $response) { - $ch = curl_init('http://appwrite-assistant:3003/'); + $ch = curl_init('http://appwrite-assistant:3003/v1/models/assistant/prompt'); $responseHeaders = []; $query = json_encode(['prompt' => $prompt]); $headers = ['accept: text/event-stream']; diff --git a/docker-compose.yml b/docker-compose.yml index 3bdbd969b3..2fb19f7126 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -869,7 +869,7 @@ services: appwrite-assistant: container_name: appwrite-assistant - image: appwrite/assistant:0.6.0 + image: appwrite/assistant:0.7.0 networks: - appwrite environment: From 06c09503df70bb2e5677cf4d390864fc4f718ce0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 04:27:28 +0000 Subject: [PATCH 16/21] update project rsource --- app/worker.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/worker.php b/app/worker.php index 6fdb49d35a..06dc4c44d8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -62,7 +62,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $project; } - return $dbForPlatform->getDocument('projects', $project->getId()); + // NOT all workers need valid DB config so we return empty if this fails + try { + return $dbForPlatform->getDocument('projects', $project->getId()); + } catch (\Throwable $e) { + return new Document([]); + } + }, ['message', 'dbForPlatform']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { From c7713f58dd4b0fc33f4d083a183d5ea925ac1233 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 13:52:58 +0000 Subject: [PATCH 17/21] remove worker hook --- app/worker.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/app/worker.php b/app/worker.php index 06dc4c44d8..fd9681b37e 100644 --- a/app/worker.php +++ b/app/worker.php @@ -380,20 +380,6 @@ try { $worker = $platform->getWorker(); -$worker - ->init() - ->inject('project') - ->inject('dbForConsole') - ->action(function (Document $project, Database $dbForConsole) { - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); - } - } - }); - $worker ->shutdown() ->inject('pools') From 0cdb2a98be906e86e6248ef78394c32ae9dcf48a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:05:37 +0000 Subject: [PATCH 18/21] fix db name --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 2c43116c10..bb60b01ddd 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -96,7 +96,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $accessedAt = $project->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); } } From 2fb9ebe71df40a258f6c89d8d16a0ba8fc99f331 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:05:49 +0000 Subject: [PATCH 19/21] update project accessed at in schedules --- .../Platform/Tasks/ScheduleExecutions.php | 12 ++++++++++++ src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 15 +++++++++++++-- src/Appwrite/Platform/Tasks/ScheduleMessages.php | 14 +++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 50beb48e9d..9be7950fd6 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -5,6 +5,8 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; use Swoole\Coroutine as Co; use Utopia\Database\Database; +use Utopia\Database\DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleExecutions extends ScheduleBase @@ -57,6 +59,16 @@ class ScheduleExecutions extends ScheduleBase $delay = $scheduledAt->getTimestamp() - (new \DateTime())->getTimestamp(); + $project = $schedule['project']; + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + \go(function () use ($queueForFunctions, $schedule, $delay, $data) { Co::sleep($delay); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 156ff1e31d..d1a9662967 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -7,6 +7,7 @@ use Cron\CronExpression; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase @@ -70,7 +71,7 @@ class ScheduleFunctions extends ScheduleBase } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools) { + \go(function () use ($delay, $scheduleKeys, $pools, $dbForPlatform) { \sleep($delay); // in seconds $queue = $pools->get('queue')->pop(); @@ -84,6 +85,16 @@ class ScheduleFunctions extends ScheduleBase $schedule = $this->schedules[$scheduleKey]; + $project = $schedule['project']; + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + $queueForFunctions = new Func($connection); $queueForFunctions @@ -91,7 +102,7 @@ class ScheduleFunctions extends ScheduleBase ->setFunction($schedule['resource']) ->setMethod('POST') ->setPath('/') - ->setProject($schedule['project']) + ->setProject($project) ->trigger(); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 2136e62f08..3beb8c48fb 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -4,6 +4,8 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Messaging; use Utopia\Database\Database; +use Utopia\Database\DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleMessages extends ScheduleBase @@ -45,10 +47,20 @@ class ScheduleMessages extends ScheduleBase $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); + $project = $schedule['project']; + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + $queueForMessaging ->setType(MESSAGE_SEND_TYPE_EXTERNAL) ->setMessageId($schedule['resourceId']) - ->setProject($schedule['project']) + ->setProject($project) ->trigger(); $dbForPlatform->deleteDocument( From d7d69b63a8f95eefc895a532079fbc56ea741045 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:15:27 +0000 Subject: [PATCH 20/21] reset worker --- app/worker.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/app/worker.php b/app/worker.php index fd9681b37e..563a801b6b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -62,13 +62,7 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $project; } - // NOT all workers need valid DB config so we return empty if this fails - try { - return $dbForPlatform->getDocument('projects', $project->getId()); - } catch (\Throwable $e) { - return new Document([]); - } - + return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { @@ -435,4 +429,4 @@ $worker->workerStart() Console::info("Worker $workerName started"); }); -$worker->start(); +$worker->start(); \ No newline at end of file From e24cce4e48aee24b6fc8e6713565e82bf1b340d1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:19:26 +0000 Subject: [PATCH 21/21] refactor --- app/worker.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 12 ++++++++++++ src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 12 +----------- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 13 ++----------- src/Appwrite/Platform/Tasks/ScheduleMessages.php | 14 ++------------ 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/app/worker.php b/app/worker.php index 563a801b6b..2c7d8acb22 100644 --- a/app/worker.php +++ b/app/worker.php @@ -429,4 +429,4 @@ $worker->workerStart() Console::info("Worker $workerName started"); }); -$worker->start(); \ No newline at end of file +$worker->start(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 10623e7fa5..dad2db0d9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -9,6 +9,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Pools\Group; use Utopia\System\System; @@ -39,6 +40,17 @@ abstract class ScheduleBase extends Action ->callback(fn (Group $pools, Database $dbForPlatform, callable $getProjectDB) => $this->action($pools, $dbForPlatform, $getProjectDB)); } + protected function updateProjectAccess(Document $project, Database $dbForPlatform): void + { + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + } + /** * 1. Load all documents from 'schedules' collection to create local copy * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 9be7950fd6..086bad513e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -5,8 +5,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; use Swoole\Coroutine as Co; use Utopia\Database\Database; -use Utopia\Database\DateTime; -use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleExecutions extends ScheduleBase @@ -59,15 +57,7 @@ class ScheduleExecutions extends ScheduleBase $delay = $scheduledAt->getTimestamp() - (new \DateTime())->getTimestamp(); - $project = $schedule['project']; - - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); - } - } + $this->updateProjectAccess($schedule['project'], $dbForPlatform); \go(function () use ($queueForFunctions, $schedule, $delay, $data) { Co::sleep($delay); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index d1a9662967..c443bb6c2d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -7,7 +7,6 @@ use Cron\CronExpression; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; -use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase @@ -85,15 +84,7 @@ class ScheduleFunctions extends ScheduleBase $schedule = $this->schedules[$scheduleKey]; - $project = $schedule['project']; - - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); - } - } + $this->updateProjectAccess($schedule['project'], $dbForPlatform); $queueForFunctions = new Func($connection); @@ -102,7 +93,7 @@ class ScheduleFunctions extends ScheduleBase ->setFunction($schedule['resource']) ->setMethod('POST') ->setPath('/') - ->setProject($project) + ->setProject($schedule['project']) ->trigger(); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 3beb8c48fb..5d997fc5bb 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -4,8 +4,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Messaging; use Utopia\Database\Database; -use Utopia\Database\DateTime; -use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleMessages extends ScheduleBase @@ -47,20 +45,12 @@ class ScheduleMessages extends ScheduleBase $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); - $project = $schedule['project']; - - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); - } - } + $this->updateProjectAccess($schedule['project'], $dbForPlatform); $queueForMessaging ->setType(MESSAGE_SEND_TYPE_EXTERNAL) ->setMessageId($schedule['resourceId']) - ->setProject($project) + ->setProject($schedule['project']) ->trigger(); $dbForPlatform->deleteDocument(