diff --git a/composer.json b/composer.json index fa20fa87e8..fe3400cfe5 100644 --- a/composer.json +++ b/composer.json @@ -91,12 +91,6 @@ "laravel/pint": "^1.14", "phpbench/phpbench": "^1.2" }, - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/utopia-php/queue" - } - ], "provide": { "ext-phpiredis": "*" }, diff --git a/composer.lock b/composer.lock index 6cc6d22849..0af2bc4795 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7fbf719806f233b49b04da95764cd06d", + "content-hash": "232691925e05350c7a3831a4e43d79d1", "packages": [ { "name": "adhocore/jwt", @@ -4490,16 +4490,16 @@ }, { "name": "utopia-php/queue", - "version": "0.8.1", + "version": "0.8.2", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "dc2654273f61d493f4548bdb7061a59e6935e883" + "reference": "a6ec26a787e8292ca2d7b8f5a0ad179b46b2c4d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/dc2654273f61d493f4548bdb7061a59e6935e883", - "reference": "dc2654273f61d493f4548bdb7061a59e6935e883", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/a6ec26a787e8292ca2d7b8f5a0ad179b46b2c4d0", + "reference": "a6ec26a787e8292ca2d7b8f5a0ad179b46b2c4d0", "shasum": "" }, "require": { @@ -4528,25 +4528,7 @@ "Utopia\\Queue\\": "src/Queue" } }, - "autoload-dev": { - "psr-4": { - "Tests\\E2E\\": "tests/Queue/E2E" - } - }, - "scripts": { - "test": [ - "phpunit" - ], - "analyse": [ - "vendor/bin/phpstan analyse" - ], - "format": [ - "vendor/bin/pint" - ], - "lint": [ - "vendor/bin/pint --test" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -4558,18 +4540,18 @@ ], "description": "A powerful task queue.", "keywords": [ + "Tasks", "framework", "php", "queue", - "tasks", "upf", "utopia" ], "support": { - "source": "https://github.com/utopia-php/queue/tree/0.8.1", - "issues": "https://github.com/utopia-php/queue/issues" + "issues": "https://github.com/utopia-php/queue/issues", + "source": "https://github.com/utopia-php/queue/tree/0.8.2" }, - "time": "2025-02-06T09:28:06+00:00" + "time": "2025-02-06T11:01:15+00:00" }, { "name": "utopia-php/registry", diff --git a/src/Appwrite/Auth/OAuth2/Slack.php b/src/Appwrite/Auth/OAuth2/Slack.php index 8898f4d1f7..9c87e45ed6 100644 --- a/src/Appwrite/Auth/OAuth2/Slack.php +++ b/src/Appwrite/Auth/OAuth2/Slack.php @@ -20,10 +20,9 @@ class Slack extends OAuth2 * @var array */ protected array $scopes = [ - 'identity.avatar', - 'identity.basic', - 'identity.email', - 'identity.team' + 'openid', + 'email', + 'profile' ]; /** @@ -35,14 +34,15 @@ class Slack extends OAuth2 } /** + * @link https://api.slack.com/authentication/oauth-v2 + * * @return string */ public function getLoginURL(): string { - // https://api.slack.com/docs/oauth#step_1_-_sending_users_to_authorize_and_or_install - return 'https://slack.com/oauth/authorize?' . \http_build_query([ + return 'https://slack.com/oauth/v2/authorize?' . \http_build_query([ 'client_id' => $this->appID, - 'scope' => \implode(' ', $this->getScopes()), + 'user_scope' => \implode(' ', $this->getScopes()), 'redirect_uri' => $this->callback, 'state' => \json_encode($this->state) ]); @@ -56,16 +56,15 @@ class Slack extends OAuth2 protected function getTokens(string $code): array { if (empty($this->tokens)) { - // https://api.slack.com/docs/oauth#step_3_-_exchanging_a_verification_code_for_an_access_token $this->tokens = \json_decode($this->request( 'GET', - 'https://slack.com/api/oauth.access?' . \http_build_query([ + 'https://slack.com/api/oauth.v2.access?' . \http_build_query([ 'client_id' => $this->appID, 'client_secret' => $this->appSecret, 'code' => $code, 'redirect_uri' => $this->callback ]) - ), true); + ), true)['authed_user'] ?? []; } return $this->tokens; @@ -80,13 +79,13 @@ class Slack extends OAuth2 { $this->tokens = \json_decode($this->request( 'GET', - 'https://slack.com/api/oauth.access?' . \http_build_query([ + 'https://slack.com/api/oauth.v2.access?' . \http_build_query([ 'client_id' => $this->appID, 'client_secret' => $this->appSecret, 'refresh_token' => $refreshToken, 'grant_type' => 'refresh_token' ]) - ), true); + ), true)['authed_user'] ?? []; if (empty($this->tokens['refresh_token'])) { $this->tokens['refresh_token'] = $refreshToken; @@ -161,9 +160,9 @@ class Slack extends OAuth2 if (empty($this->user)) { $user = $this->request( 'GET', - 'https://slack.com/api/users.identity?token=' . \urlencode($accessToken) + 'https://slack.com/api/users.identity', + ['Authorization: Bearer ' . \urlencode($accessToken)] ); - $this->user = \json_decode($user, true); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index abcad8c02e..dad2db0d9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -11,7 +11,7 @@ use Utopia\Database\Exception; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; -use Utopia\Queue\Publisher; +use Utopia\Pools\Group; use Utopia\System\System; use function Swoole\Coroutine\run; @@ -26,7 +26,7 @@ abstract class ScheduleBase extends Action abstract public static function getName(): string; abstract public static function getSupportedResource(): string; abstract public static function getCollectionId(): string; - abstract protected function enqueueResources(Publisher $publisher, Database $dbForPlatform, callable $getProjectDB): void; + abstract protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void; public function __construct() { @@ -34,10 +34,10 @@ abstract class ScheduleBase extends Action $this ->desc("Execute {$type}s scheduled in Appwrite") - ->inject('publisher') + ->inject('pools') ->inject('dbForPlatform') ->inject('getProjectDB') - ->callback(fn (Publisher $publisher, Database $dbForPlatform, callable $getProjectDB) => $this->action($publisher, $dbForPlatform, $getProjectDB)); + ->callback(fn (Group $pools, Database $dbForPlatform, callable $getProjectDB) => $this->action($pools, $dbForPlatform, $getProjectDB)); } protected function updateProjectAccess(Document $project, Database $dbForPlatform): void @@ -56,7 +56,7 @@ abstract class ScheduleBase extends Action * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutine sleeps until exact time before sending request to worker. */ - public function action(Publisher $publisher, Database $dbForPlatform, callable $getProjectDB): void + public function action(Group $pools, Database $dbForPlatform, callable $getProjectDB): void { Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1'); Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started'); @@ -125,15 +125,17 @@ abstract class ScheduleBase extends Action $latestDocument = \end($results); } + $pools->reclaim(); + Console::success("{$total} resources were loaded in " . (\microtime(true) - $loadStart) . " seconds"); Console::success("Starting timers at " . DateTime::now()); - run(function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $publisher, $getProjectDB) { + run(function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools, $getProjectDB) { /** * The timer synchronize $schedules copy with database collection. */ - Timer::tick(static::UPDATE_TIMER * 1000, function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule) { + Timer::tick(static::UPDATE_TIMER * 1000, function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools) { $time = DateTime::now(); $timerStart = \microtime(true); @@ -182,15 +184,17 @@ abstract class ScheduleBase extends Action $lastSyncUpdate = $time; $timerEnd = \microtime(true); + $pools->reclaim(); + Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds"); }); Timer::tick( static::ENQUEUE_TIMER * 1000, - fn () => $this->enqueueResources($publisher, $dbForPlatform, $getProjectDB) + fn () => $this->enqueueResources($pools, $dbForPlatform, $getProjectDB) ); - $this->enqueueResources($publisher, $dbForPlatform, $getProjectDB); + $this->enqueueResources($pools, $dbForPlatform, $getProjectDB); }); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 25de721b38..7cd76b480d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -5,7 +5,7 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; use Swoole\Coroutine as Co; use Utopia\Database\Database; -use Utopia\Queue\Publisher; +use Utopia\Pools\Group; class ScheduleExecutions extends ScheduleBase { @@ -27,9 +27,11 @@ class ScheduleExecutions extends ScheduleBase return 'executions'; } - protected function enqueueResources(Publisher $publisher, Database $dbForPlatform, callable $getProjectDB): void + protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void { - $queueForFunctions = new Func($publisher); + $queue = $pools->get('publisher')->pop(); + $connection = $queue->getResource(); + $queueForFunctions = new Func($connection); $intervalEnd = (new \DateTime())->modify('+' . self::ENQUEUE_TIMER . ' seconds'); foreach ($this->schedules as $schedule) { @@ -81,5 +83,7 @@ class ScheduleExecutions extends ScheduleBase unset($this->schedules[$schedule['$internalId']]); } + + $queue->reclaim(); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 4f8579862f..5b8e3027a7 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -7,7 +7,7 @@ use Cron\CronExpression; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; -use Utopia\Queue\Publisher; +use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase { @@ -31,7 +31,7 @@ class ScheduleFunctions extends ScheduleBase return 'functions'; } - protected function enqueueResources(Publisher $publisher, Database $dbForPlatform, callable $getProjectDB): void + protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void { $timerStart = \microtime(true); $time = DateTime::now(); @@ -70,9 +70,12 @@ class ScheduleFunctions extends ScheduleBase } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $publisher, $dbForPlatform) { + \go(function () use ($delay, $scheduleKeys, $pools, $dbForPlatform) { \sleep($delay); // in seconds + $queue = $pools->get('publisher')->pop(); + $connection = $queue->getResource(); + foreach ($scheduleKeys as $scheduleKey) { // Ensure schedule was not deleted if (!\array_key_exists($scheduleKey, $this->schedules)) { @@ -83,7 +86,8 @@ class ScheduleFunctions extends ScheduleBase $this->updateProjectAccess($schedule['project'], $dbForPlatform); - $queueForFunctions = new Func($publisher); + $queueForFunctions = new Func($connection); + $queueForFunctions ->setType('schedule') ->setFunction($schedule['resource']) @@ -92,6 +96,8 @@ class ScheduleFunctions extends ScheduleBase ->setProject($schedule['project']) ->trigger(); } + + $queue->reclaim(); }); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index a575fb819a..201d5eab53 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -4,7 +4,7 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Messaging; use Utopia\Database\Database; -use Utopia\Queue\Publisher; +use Utopia\Pools\Group; class ScheduleMessages extends ScheduleBase { @@ -26,7 +26,7 @@ class ScheduleMessages extends ScheduleBase return 'messages'; } - protected function enqueueResources(Publisher $publisher, Database $dbForPlatform, callable $getProjectDB): void + protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void { foreach ($this->schedules as $schedule) { if (!$schedule['active']) { @@ -40,9 +40,13 @@ class ScheduleMessages extends ScheduleBase continue; } - \go(function () use ($schedule, $publisher, $dbForPlatform) { + \go(function () use ($schedule, $pools, $dbForPlatform) { + $queue = $pools->get('publisher')->pop(); + $connection = $queue->getResource(); + $queueForMessaging = new Messaging($connection); + $this->updateProjectAccess($schedule['project'], $dbForPlatform); - $queueForMessaging = new Messaging($publisher); + $queueForMessaging ->setType(MESSAGE_SEND_TYPE_EXTERNAL) ->setMessageId($schedule['resourceId']) @@ -54,6 +58,8 @@ class ScheduleMessages extends ScheduleBase $schedule['$id'], ); + $queue->reclaim(); + unset($this->schedules[$schedule['$internalId']]); }); } diff --git a/src/Appwrite/Utopia/Request.php b/src/Appwrite/Utopia/Request.php index f8c0439293..480fce58b0 100644 --- a/src/Appwrite/Utopia/Request.php +++ b/src/Appwrite/Utopia/Request.php @@ -2,8 +2,10 @@ namespace Appwrite\Utopia; +use Appwrite\Auth\Auth; use Appwrite\Utopia\Request\Filter; use Swoole\Http\Request as SwooleRequest; +use Utopia\Database\Validator\Authorization; use Utopia\Route; use Utopia\Swoole\Request as UtopiaRequest; @@ -180,4 +182,27 @@ class Request extends UtopiaRequest $headers = $this->getHeaders(); return $headers[$key] ?? $default; } + + /** + * Get User Agent + * + * Method for getting User Agent. Preferring forwarded agent for privileged users; otherwise returns default. + * + * @param string $default + * @return string + */ + public function getUserAgent(string $default = ''): string + { + $forwardedUserAgent = $this->getHeader('x-forwarded-user-agent'); + if (!empty($forwardedUserAgent)) { + $roles = Authorization::getRoles(); + $isAppUser = Auth::isAppUser($roles); + + if ($isAppUser) { + return $forwardedUserAgent; + } + } + + return UtopiaRequest::getUserAgent($default); + } } diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index cca27cc3be..788f949fb3 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2307,6 +2307,60 @@ class AccountCustomClientTest extends Scope $this->assertNotEmpty($response['body']['$id']); $this->assertNotEmpty($response['body']['expire']); $this->assertEmpty($response['body']['secret']); + $this->assertEquals('browser', $response['body']['clientType']); + $this->assertEquals('CH', $response['body']['clientCode']); + $this->assertEquals('Chrome', $response['body']['clientName']); + + // Forwarded User Agent with API Key + $response = $this->client->call(Client::METHOD_POST, '/users/' . $data['id'] . '/tokens', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'expire' => 60 + ]); + + $userId = $response['body']['userId']; + $secret = $response['body']['secret']; + + $response = $this->client->call(Client::METHOD_POST, '/account/sessions/token', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-forwarded-user-agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36' + ], [ + 'userId' => $userId, + 'secret' => $secret + ]); + + $this->assertEquals('browser', $response['body']['clientType']); + $this->assertEquals('CM', actual: $response['body']['clientCode']); + $this->assertEquals('Chrome Mobile', $response['body']['clientName']); + + // Forwarded User Agent without API Key + $response = $this->client->call(Client::METHOD_POST, '/users/' . $data['id'] . '/tokens', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'expire' => 60 + ]); + + $userId = $response['body']['userId']; + $secret = $response['body']['secret']; + + $response = $this->client->call(Client::METHOD_POST, '/account/sessions/token', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-forwarded-user-agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36' + ], [ + 'userId' => $userId, + 'secret' => $secret + ]); + + $this->assertEquals('browser', $response['body']['clientType']); + $this->assertEquals('CH', $response['body']['clientCode']); + $this->assertEquals('Chrome', $response['body']['clientName']); /** * Test for FAILURE